chore(main): release 1.34.0 (#456)

* chore(main): release 1.34.0

* Apply automatic changes

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -1,3 +1,2 @@
-SITE_URL=localhost
 DB_PASSWORD=changeme
-POSTGRES_VERSION=13.3.0
+WM_BASE_URL=windmill.localhost

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+*       @rubenfiszel
+
+/community/ @fatonramadani @rubenfiszel
+/frontend/ @fatonramadani @rubenfiszel

.github/DockerfileBackendTests

@@ -0,0 +1,62 @@
+FROM python:3.10-slim-buster as nsjail
+
+WORKDIR /nsjail
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    bison=2:3.3.* \
+    flex=2.6.* \
+    g++=4:8.3.* \
+    gcc=4:8.3.* \
+    git=1:2.20.* \
+    libprotobuf-dev=3.6.* \
+    libnl-route-3-dev=3.4.* \
+    make=4.2.* \
+    pkg-config=0.29-6 \
+    protobuf-compiler=3.6.*
+
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+RUN make
+
+
+FROM rust:slim-buster as builder
+
+RUN apt-get update && apt-get install -y git libssl-dev pkg-config
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    curl lld
+
+ENV SQLX_OFFLINE=true
+
+COPY ./nsjail /nsjail
+
+RUN mkdir -p /frontend/build
+
+RUN apt-get update \
+    && apt-get install -y ca-certificates tzdata libpq5 \
+    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
+    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
+    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
+    libv8-dev tesseract-ocr \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV TZ=Etc/UTC
+
+ENV PYTHON_VERSION 3.10.4
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
+    && make -j 4 && make install
+
+RUN /usr/local/bin/python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install nltk
+RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
+
+COPY --from=nsjail /nsjail/nsjail /bin/nsjail
+
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
+RUN apt-get update \
+    && apt-get install -y postgresql-client

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [rubenfiszel]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +1,27 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
+title: 'bug:'
+labels: 'bug'
+assignees: 'rubenfiszel'
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
+**Describe the bug** A clear and concise description of what the bug is.
+
+**To Reproduce** Steps to reproduce the behavior:
 
-**To Reproduce**
-Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior**
-A clear and concise description of what you expected to happen.
+**Expected behavior** A clear and concise description of what you expected to
+happen.
 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
+**Screenshots** If applicable, add screenshots to help explain your problem.
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+**Windmill version** Go on the left menu -> <user> -> User Settings and copy the
+printed version in "Running windmill version (backend): XXX".
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.
+**Additional context** Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,8 @@
+---
+name: Feature Request
+about: Create a feature request
+title: 'feature: '
+labels: 'feature'
+assignees: 'rubenfiszel'
+
+---

.github/change-versions.sh

@@ -5,12 +5,15 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml
+sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" Pipfile
-sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" Pipfile
+# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
+sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
+sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
 
 sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" backend/Cargo.lock
+
+cd frontend && npm i --package-lock-only

.github/dependabot.yml

@@ -0,0 +1,39 @@
+# Basic set up for three package managers
+
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for npm
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for cargo
+  - package-ecosystem: "cargo"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for Docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for wmill python client
+  - package-ecosystem: "pip"
+    directory: "/python-client/wmill"
+    schedule:
+      interval: "weekly"
+      
+  # Maintain dependencies for wmill_pg python client
+  - package-ecosystem: "pip"
+    directory: "/python-client/wmill_pg"
+    schedule:
+      interval: "weekly"

.github/pull_hub_items.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+RT=$(curl -s https://hub.windmill.dev/resource_types/list | jq -c -r '.[]')
+for item in ${RT[@]}; do
+    name=$(jq -r '.name' <<< "$item")
+    id=$(jq -r '.id' <<< "$item")
+    echo $name $id 
+    body=$(curl -s -H "accept: application/json" https://hub.windmill.dev/resource_types/${id}/${name})
+    jq -r '.resource_type.schema' <<< "$body" > ./tmp
+    description=$(jq -r '.resource_type.description' <<< "$body")
+    description=$(echo -E $description)
+    echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
+    rm ./tmp
+done

.github/workflows/automerge-dependabot.yml

@@ -0,0 +1,26 @@
+name: dependabot auto-merge
+
+on: pull_request_target
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.3.3
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Enable auto-merge for Dependabot PRs
+        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
+        run: |
+          echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
+          gh pr review --approve "$PR_URL"
+          gh pr merge --auto --squash "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/backend-test.yml

@@ -0,0 +1,41 @@
+name: Backend only integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "backend/**"
+      - ".github/workflows/backend-test.yml"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "backend/**"
+      - ".github/workflows/backend-test.yml"
+
+jobs:
+  cargo_test:
+    runs-on: [self-hosted, new]
+    container:
+      image: ghcr.io/windmill-labs/backend-tests
+      options: --privileged
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend -> target
+      - name: cargo test
+        timeout-minutes: 5
+        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test -- --nocapture

.github/workflows/change-versions.yml

@@ -7,8 +7,9 @@ on:
 jobs:
   change_version:
     runs-on: ubuntu-latest
+    container: node:18
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
       - uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/deno_on_release.yml

@@ -0,0 +1,49 @@
+name: Publish deno-client
+on:
+  push:
+    tags:
+      - "v*"
+env:
+  repo: windmill-deno-client
+
+jobs:
+  build_deno_and_push_to_repo:
+    runs-on: ubuntu-latest
+    container: openapitools/openapi-generator-cli:v6.0.0-beta
+    steps:
+      - uses: actions/checkout@v3
+      - name: generate_deno
+        run: |
+          cd deno-client
+          rm .gitignore
+          ./generate.sh
+      - name: Pushes to another repository
+        id: push_directory
+        uses: cpina/github-action-push-to-another-repository@devel
+        env:
+          API_TOKEN_GITHUB: ${{ secrets.DENO_PAT }}
+        with:
+          source-directory: deno-client/
+          destination-github-username: ${{ github.repository_owner }}
+          destination-repository-name: ${{ env.repo }}
+          user-email: ruben@windmill.dev
+          commit-message: See ORIGIN_COMMIT from $GITHUB_REF
+          target-branch: main
+
+  tag_repo:
+    needs: [build_deno_and_push_to_repo]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository_owner }}/${{ env.repo }}
+          token: ${{ secrets.DENO_PAT }}
+          path: ./client
+
+      - name: Push client
+        run: |
+          cd ./client
+          git config --global user.email "ruben@windmill.dev"
+          git config --global user.name "rubenfiszel[bot]"
+          git tag -a ${{ github.ref_name }} -m "${{ github.ref_name }}"
+          git push --tags

.github/workflows/deploy_to_windmill.yml

@@ -3,6 +3,8 @@ name: Deploy to windmill.dev
 on:
   push:
     branches: [main]
+    paths:
+      - "community/**"
 
 jobs:
   deploy:
@@ -10,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v1.0.0
+        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
         with:
           dry_run: false
           input_dir: community

.github/workflows/docker-image.yml

@@ -1,7 +1,13 @@
-name: Docker Image CI
+env:
+  LOCAL_REGISTRY: registry.wimill.xyz
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build and push docker image
 on:
   push:
     branches: [main]
+    tags: ["*"]
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -12,28 +18,94 @@ concurrency:
 jobs:
   build:
     runs-on: [self-hosted, new]
-    env:
-      DOCKER_BUILDKIT: 1
     steps:
-      - name: Wait for release to succeed
-        if: github.ref == 'refs/heads/main'
-        uses: lewagon/wait-on-check-action@v1.0.0
-        with:
-          ref: ${{ github.ref }}
-          check-name: "Release please"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 10
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: deploy staging stack
-        run: |
-          docker build . --cache-from "registry.wimill.xyz/windmill:staging" -t "registry.wimill.xyz/windmill:staging" --build-arg BUILDKIT_INLINE_CACHE=1
-          docker push "registry.wimill.xyz/windmill:staging"
-      - name: deploy demo stack
-        if: github.ref == 'refs/heads/main'
-        run: |
-          docker tag registry.wimill.xyz/windmill:staging registry.wimill.xyz/windmill:main
-          docker push registry.wimill.xyz/windmill:main
-      # - name: pruning unused images
-      #   run: sudo docker image prune -a
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker meta local
+        id: metalocal
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push privately
+        uses: docker/build-push-action@v3
+        if: github.event_name == 'pull_request'
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ steps.metalocal.outputs.tags }}
+
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+
+      - name: Docker meta
+        if: github.event_name != 'pull_request'
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Login to registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly
+        uses: docker/build-push-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+            ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+  playwright:
+    runs-on: [self-hosted, new]
+    needs: [build]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_USER: admin
+          POSTGRES_PASSWORD: changeme
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Docker"
+        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+        id: docker-container
+      - name: "Playwright run"
+        timeout-minutes: 2
+        run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
+      - name: "Clean up"
+        run: docker kill ${{ steps.docker-container.outputs.id }}
+        if: always()

.github/workflows/on-release.yml

@@ -1,38 +0,0 @@
-name: Build LSP Docker
-on:
-  push:
-    branches: [main]
-    paths:
-      - "python-client/**"
-      - "Pipfile"
-      - ".github/workflows/on-release.yml"
-
-jobs:
-  build_lsp:
-    runs-on: [self-hosted, new]
-    steps:
-      - name: Wait for release to succeed
-        if: github.ref == 'refs/heads/main'
-        uses: lewagon/wait-on-check-action@v1.0.0
-        with:
-          ref: ${{ github.ref }}
-          check-name: "Release please"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 10
-      - uses: actions/checkout@v2
-      - name: Upload python client
-        env:
-          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          cd python-client
-          export PATH=$PATH:/usr/local/bin
-          export PATH=$PATH:/root/.local/bin
-          ./publish.sh
-      - name: Build the Docker image
-        run: |
-          cd lsp
-          sudo docker pull "registry.wimill.xyz/lsp:main" || true
-          sudo docker build  .  --cache-from "registry.wimill.xyz/lsp:main" -t "registry.wimill.xyz/lsp:main" --build-arg BUILDKIT_INLINE_CACHE=1
-      - name: push to registry
-        run: |
-          sudo docker push "registry.wimill.xyz/lsp:main"

.github/workflows/pull-hub.yml

@@ -0,0 +1,20 @@
+name: Pull Hub Items
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 0 */1 * *"
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  change_version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Pull hub
+        run: ./.github/pull_hub_items.sh
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          title: sync hub items with community
+          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -0,0 +1,65 @@
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-lsp
+
+name: Publish python-client
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+jobs:
+  publish_pypi:
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Upload python client
+        env:
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          cd python-client
+          export PATH=$PATH:/usr/local/bin
+          export PATH=$PATH:/root/.local/bin
+          ./publish.sh
+
+  publish_lsp:
+    needs: [publish_pypi]
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:lsp"
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+            ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/release-please.yml

@@ -1,14 +1,14 @@
 on:
   push:
-    branches:
-      - main
+    branches: [main]
+
 name: release-please
 jobs:
   release-please:
     name: "Release please"
     runs-on: ubuntu-latest
     steps:
-      - uses: GoogleCloudPlatform/release-please-action@v2
+      - uses: GoogleCloudPlatform/release-please-action@v3
         with:
           release-type: simple
           package-name: windmill

.github/workflows/sign-cla.yml

@@ -0,0 +1,34 @@
+name: "CLA Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened, closed, synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "CLA Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        # Beta Release
+        uses: cla-assistant/github-action@v2.2.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}
+        with:
+          path-to-signatures: "signatures/cla.json"
+          path-to-document: "https://github.com/windmill-labs/windmill/blob/master/CLA.md"
+          branch: "signatures"
+          allowlist: rubenfiszel,bot*
+
+          #below are the optional inputs - If the optional inputs are not given, then default values will be taken
+          #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #remote-repository-name:  enter the  remote repository name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #create-file-commit-message: 'For example: Creating file for storing CLA Signatures'
+          #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo'
+          #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign'
+          #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA'
+          #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.'
+          #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true)
+          #use-dco-flag: true - If you are using DCO instead of CLA

CHANGELOG.md

@@ -1,3 +1,636 @@
 # Changelog
 
+
+## [1.34.0](https://github.com/windmill-labs/windmill/compare/v1.33.0...v1.34.0) (2022-08-21)
+
+
+### Features
+
+* implicit types infered from default parameters ([b9dfbfa](https://github.com/windmill-labs/windmill/commit/b9dfbfa2d8d86f0313d4f8b1829c27a1b1c1c380))
+
+## [1.33.0](https://github.com/windmill-labs/windmill/compare/v1.32.0...v1.33.0) (2022-08-21)
+
+
+### Features
+
+* PostgreSQL parametrized statement handled as typescript template ([1aa28c5](https://github.com/windmill-labs/windmill/commit/1aa28c55990b27901c698eea6812a51eaafc97bb))
+
+## [1.32.0](https://github.com/windmill-labs/windmill/compare/v1.31.0...v1.32.0) (2022-08-21)
+
+
+### Features
+
+* **backend:** failure_module ([#452](https://github.com/windmill-labs/windmill/issues/452)) ([32d067f](https://github.com/windmill-labs/windmill/commit/32d067f8c078fd7940c2c4bab8dbb01de876503e))
+* **frontend:** Open/Close UI ([#445](https://github.com/windmill-labs/windmill/issues/445)) ([7e4aac9](https://github.com/windmill-labs/windmill/commit/7e4aac997175bf2ba479021742e5aa8abab4ff41))
+* private imports ([a5343fa](https://github.com/windmill-labs/windmill/commit/a5343fa959a237120fc22d6a3c06da3b29a3f990))
+* rely on PG time rather than worker time ([0057266](https://github.com/windmill-labs/windmill/commit/00572668f16183f7508b9966213cbcc9c106da51))
+
+
+### Bug Fixes
+
+* **backend:** clear_schedule only clear non running jobs ([0cd814c](https://github.com/windmill-labs/windmill/commit/0cd814cfec3ab088f7646b6b9f6970e48961e710))
+* **backend:** fixes forloop with 257 items only iterates once ([#446](https://github.com/windmill-labs/windmill/issues/446)) ([bae8573](https://github.com/windmill-labs/windmill/commit/bae85732ff7c70796c2defcd0430d64dedeb36f7))
+* **backend:** started_at info for completed_job is no more completed_at ([77a6851](https://github.com/windmill-labs/windmill/commit/77a685144ddc65c8e5205688ce7e411a14f7915b))
+* cancel a flow now does the expected behavior ([c0e9cd0](https://github.com/windmill-labs/windmill/commit/c0e9cd05641d28336cc26eee5167a397149d61f2))
+* **deno-client:** pg module now supports prepared statements ([5900a03](https://github.com/windmill-labs/windmill/commit/5900a03c045861732bbf6f7bff1280f3c94b86ce))
+* **deno-client:** wrap the deno-postgres client and not the query statement ([68aaf32](https://github.com/windmill-labs/windmill/commit/68aaf3267ce183e366696ebadc644580976ed7ce))
+* **frontend:** Fix loops pickable properties ([#441](https://github.com/windmill-labs/windmill/issues/441)) ([0681472](https://github.com/windmill-labs/windmill/commit/068147251c831d3ab8564ccb909ad72ef2e32e74))
+* **frontend:** input checks refresh when schema change ([15f7cad](https://github.com/windmill-labs/windmill/commit/15f7cadc3d179993b70e1f7584d532528aaabb52))
+* **frontend:** link to schedule in runs discriminate isFlows ([7d76e69](https://github.com/windmill-labs/windmill/commit/7d76e69be9753cc572ce7c085d0191a31471d9e9))
+* **frontend:** simplify flow preview  logic([#450](https://github.com/windmill-labs/windmill/issues/450)) ([bc5a568](https://github.com/windmill-labs/windmill/commit/bc5a5688ce9c351ad745be225c11a977c1ad2afb))
+* handle 0 length for-loops in the backend ([#440](https://github.com/windmill-labs/windmill/issues/440)) ([561e13e](https://github.com/windmill-labs/windmill/commit/561e13e51ee7ffcf20bc524c22d756ea582d546e))
+* restart zombie jobs was restarting all jobs ([da77d04](https://github.com/windmill-labs/windmill/commit/da77d040942c01b0011e76546dddd6aaa7786b8f))
+
+## [1.31.0](https://github.com/windmill-labs/windmill/compare/v1.30.0...v1.31.0) (2022-08-17)
+
+
+### Features
+
+* allow to configure port via envar ([#407](https://github.com/windmill-labs/windmill/issues/407)) ([34be056](https://github.com/windmill-labs/windmill/commit/34be0564f89f942478c25e77fd77a515367a6afd))
+* db users: admin -> windmill_admin, app -> windmill_user ([#404](https://github.com/windmill-labs/windmill/issues/404)) ([1c40f01](https://github.com/windmill-labs/windmill/commit/1c40f01e5d8e3d854de4c30d9f5e4f731c220ce2))
+* **frontend:** Redesign of the Flow Editor + Arbitrary forloop ([127b0b4](https://github.com/windmill-labs/windmill/commit/127b0b4e5e6a96f91d7e8234cc52d887afb637b0))
+
+
+### Bug Fixes
+
+* **backend:** collecting result when for loop is not the last step [#422](https://github.com/windmill-labs/windmill/issues/422)  ([e606118](https://github.com/windmill-labs/windmill/commit/e6061189438fb3a7e630d2e390075fc3eded984c))
+* **self-hosting:** add lsp and caddy to docke-compose ([#432](https://github.com/windmill-labs/windmill/issues/432)) ([1004518](https://github.com/windmill-labs/windmill/commit/100451878c26d2fa324c6195838accae959a5310))
+* set secure only for https ([1275f5f](https://github.com/windmill-labs/windmill/commit/1275f5f7fb65e32a17d7d397d43d0b49ecd5cd0e))
+* users privileges ([2bdb617](https://github.com/windmill-labs/windmill/commit/2bdb617b1f80104bd3314656603dccb0021e05cb))
+
+## [1.30.0](https://github.com/windmill-labs/windmill/compare/v1.29.0...v1.30.0) (2022-08-13)
+
+
+### Features
+
+* add literal object type support ([#401](https://github.com/windmill-labs/windmill/issues/401)) ([845de82](https://github.com/windmill-labs/windmill/commit/845de8206214ed265aef895f0d13636e6e0e26ce))
+* support union type will null | undefined ([#400](https://github.com/windmill-labs/windmill/issues/400)) ([0384727](https://github.com/windmill-labs/windmill/commit/0384727a56347aa01a5fee06c82bd49eab2522fa))
+* support union types ([#398](https://github.com/windmill-labs/windmill/issues/398)) ([e68ea1b](https://github.com/windmill-labs/windmill/commit/e68ea1b8fc4f88e587121387ecac6858d04ebae2))
+
+## [1.29.0](https://github.com/windmill-labs/windmill/compare/v1.28.1...v1.29.0) (2022-08-10)
+
+
+### Features
+
+* _value, _index => iter.value, iter.index ([07f4a21](https://github.com/windmill-labs/windmill/commit/07f4a217d0c6b46fd3defaa0242d229a60c69463))
+* remove res1 wrapping ([e76a981](https://github.com/windmill-labs/windmill/commit/e76a9816ee09e59d5c38bf0c19231bac8347148c))
+
+
+### Bug Fixes
+
+* do not skip undefined values ([8b68a87](https://github.com/windmill-labs/windmill/commit/8b68a87c523fe13a9f45786ee0fbb57b10efda13))
+* **python:** not filled field with default <function_call> now call the default function ([33962c4](https://github.com/windmill-labs/windmill/commit/33962c44660fd20173a0ae14b00a66a985dd4fc7))
+* surface new _iterator value ([13b1904](https://github.com/windmill-labs/windmill/commit/13b1904a7ab5a6e7a7c82d2a2806648441759756))
+* update logs even if last new log was < 500ms ([c69621f](https://github.com/windmill-labs/windmill/commit/c69621fa7a9a18223e7854f0824bd6fbcabdfe10))
+
+## [1.28.1](https://github.com/windmill-labs/windmill/compare/v1.28.0...v1.28.1) (2022-08-05)
+
+
+### Bug Fixes
+
+* **frontend:** add toggl connect ([#341](https://github.com/windmill-labs/windmill/issues/341)) ([b94895f](https://github.com/windmill-labs/windmill/commit/b94895f24eb4ba1b67f499a98c6e6e8d9d006b14))
+* **frontend:** schedule args in flow ([#343](https://github.com/windmill-labs/windmill/issues/343)) ([350a25c](https://github.com/windmill-labs/windmill/commit/350a25c837b1367fa5568dd1de0196202d632bd0))
+* improve flow viewer with retrieving hub script ([80e28db](https://github.com/windmill-labs/windmill/commit/80e28dbba3e77154c0017bd8e74d144e6aae13fb))
+
+## [1.28.0](https://github.com/windmill-labs/windmill/compare/v1.27.2...v1.28.0) (2022-08-04)
+
+
+### Features
+
+* **frontend:** global flow preview ([#329](https://github.com/windmill-labs/windmill/issues/329)) ([615f69e](https://github.com/windmill-labs/windmill/commit/615f69e935e9c9c0b60edfb6dc2e82aebba623b9))
+
+
+### Bug Fixes
+
+* **api:** add discord webhook manual instructions ([a9a4b9b](https://github.com/windmill-labs/windmill/commit/a9a4b9b21d7b68a3e46c28ce13986d7a9ebd2cac))
+* **backend:** generalize oauth clients to take in extra params ([6332910](https://github.com/windmill-labs/windmill/commit/6332910dd27f78d555f0ab040545e98dedbea89d))
+* **backend:** handle better some flow edge-cases ([3bcd542](https://github.com/windmill-labs/windmill/commit/3bcd542130bc0cb45dfb1fa7681dd4b7beb95c7e))
+* **backend:** handle better some flow edge-cases ([9885361](https://github.com/windmill-labs/windmill/commit/988536128bd04dab94cc686bc2db547e57894587))
+* **backend:** handle better some flow edge-cases ([70de6e3](https://github.com/windmill-labs/windmill/commit/70de6e3972af81aec68b706dca93e16182a584bb))
+* **backend:** prometheus histogram for worker job timer ([#312](https://github.com/windmill-labs/windmill/issues/312)) ([4055586](https://github.com/windmill-labs/windmill/commit/40555868e6221620beca85ebafad2da67e56ec08))
+* **frontend:** add jpeg support ([0e8552b](https://github.com/windmill-labs/windmill/commit/0e8552ba800f13add6b25a83a765dace8d4369e7))
+* **frontend:** loading template pick the language as well ([82c7ddc](https://github.com/windmill-labs/windmill/commit/82c7ddc00e79a1cc5336a0a219f46d705c2c8d88))
+* **frontend:** Use the bracket notation when an identifier is not a valid JS expression ([#327](https://github.com/windmill-labs/windmill/issues/327)) ([05324bd](https://github.com/windmill-labs/windmill/commit/05324bd3562f6066cdc12d74c87033325d1c7ef1))
+* **oauth2:** remove discord oauth integration ([986e76d](https://github.com/windmill-labs/windmill/commit/986e76dc8729a53d09cd83531d474f9b5fe88f35))
+
+## [1.27.2](https://github.com/windmill-labs/windmill/compare/v1.27.1...v1.27.2) (2022-08-02)
+
+
+### Bug Fixes
+
+* **deno-client:** getResource can now fetch non-object values ([b128388](https://github.com/windmill-labs/windmill/commit/b128388cc652d4cd369a88b93985a2c051003abd))
+
+## [1.27.1](https://github.com/windmill-labs/windmill/compare/v1.27.0...v1.27.1) (2022-08-02)
+
+
+### Bug Fixes
+
+* migrate to new style radio button ([893ee94](https://github.com/windmill-labs/windmill/commit/893ee941d72a7036f0ea272c49bbe5cd3eee64d5))
+
+## [1.27.0](https://github.com/windmill-labs/windmill/compare/v1.26.3...v1.27.0) (2022-08-02)
+
+
+### Features
+
+* add primitive sql format ([#320](https://github.com/windmill-labs/windmill/issues/320)) ([9daff2a](https://github.com/windmill-labs/windmill/commit/9daff2a228791234a3dd70c0ee829e284daf1592))
+
+
+### Bug Fixes
+
+* prefer `COPY` over `ADD` ([#319](https://github.com/windmill-labs/windmill/issues/319)) ([24a7e46](https://github.com/windmill-labs/windmill/commit/24a7e46fe99d5a1f7d5b22334fa5f6ce76e82d94))
+* typos ([#301](https://github.com/windmill-labs/windmill/issues/301)) ([9e84b45](https://github.com/windmill-labs/windmill/commit/9e84b458b139e86eb51dba9c5b228f141ca649b3))
+
+## [1.26.3](https://github.com/windmill-labs/windmill/compare/v1.26.2...v1.26.3) (2022-08-01)
+
+
+### Bug Fixes
+
+* displaying which group you are a member of that gave you access to item ([1bd0269](https://github.com/windmill-labs/windmill/commit/1bd026924b8a3b01f7729b627f939d8af872a483))
+* refresh jobs result when hopping from flow to flow ([c86abe6](https://github.com/windmill-labs/windmill/commit/c86abe6ae01efd519f67ead233ebddf39f1539c0))
+
+## [1.26.2](https://github.com/windmill-labs/windmill/compare/v1.26.1...v1.26.2) (2022-07-31)
+
+
+### Bug Fixes
+
+* deno api generator now supports openflow ([5b548a0](https://github.com/windmill-labs/windmill/commit/5b548a0e71669aad90343e70f3f1c9dc3a6d4baf))
+
+## [1.26.1](https://github.com/windmill-labs/windmill/compare/v1.26.0...v1.26.1) (2022-07-31)
+
+
+### Bug Fixes
+
+* encoding state now supports unicode including emojis ([6b61227](https://github.com/windmill-labs/windmill/commit/6b61227481422fe52384f6de8146388a8471ff60))
+
+## [1.26.0](https://github.com/windmill-labs/windmill/compare/v1.25.0...v1.26.0) (2022-07-29)
+
+
+### Features
+
+* resource type picker in schema modal + proper initialization of raw javascript editor when applicable ([01bb107](https://github.com/windmill-labs/windmill/commit/01bb107a0f3e3899ec99718974b2484ab5978c92))
+
+
+### Bug Fixes
+
+* forloop flows unsoundness fix part I ([1b5ce32](https://github.com/windmill-labs/windmill/commit/1b5ce3243b364d02903072a9af5e15447622e9fb))
+* small bar mode and editor nits ([4e3a02a](https://github.com/windmill-labs/windmill/commit/4e3a02a8e44e25e6b5402f732b9af6969d06dcc0))
+
+## [1.25.0](https://github.com/windmill-labs/windmill/compare/v1.24.2...v1.25.0) (2022-07-29)
+
+
+### Features
+
+* base64 support in schema editor ([2cb6e6e](https://github.com/windmill-labs/windmill/commit/2cb6e6e7021819a9aa9618436abf2f0fa5b3587b))
+
+
+### Bug Fixes
+
+* update variable and resources now return error if nothing was updated ([0faabdb](https://github.com/windmill-labs/windmill/commit/0faabdbc40b049258b074c6c20c1406ca14b8481))
+
+## [1.24.2](https://github.com/windmill-labs/windmill/compare/v1.24.1...v1.24.2) (2022-07-28)
+
+
+### Bug Fixes
+
+* get_variable refresh_token bug ([390e9b3](https://github.com/windmill-labs/windmill/commit/390e9b37fb201242ac6983c145c9de5b242f7a7b))
+* if :path is not a valid path, do not even attempt to fetch it ([6dec447](https://github.com/windmill-labs/windmill/commit/6dec4479537164fe17bea7f88fd60b1d4f42e887))
+* monaco editor fixes ([f255cc2](https://github.com/windmill-labs/windmill/commit/f255cc253fcf14850442e8d4bf64635287b88314))
+
+## [1.24.1](https://github.com/windmill-labs/windmill/compare/v1.24.0...v1.24.1) (2022-07-27)
+
+
+### Bug Fixes
+
+* encrypt the refresh token ([a051c21](https://github.com/windmill-labs/windmill/commit/a051c2121a63983f6925ce2e3a9b9deb01df2f04))
+* keep previous refresh token if no new ones were provided ([3feef73](https://github.com/windmill-labs/windmill/commit/3feef738dc145603576649a91f0ddc0e82215841))
+* skip_failures is boolean not bool ([4ca71c1](https://github.com/windmill-labs/windmill/commit/4ca71c1e5da0132724ab4c9771f5fdc590b866f8))
+
+## [1.24.0](https://github.com/windmill-labs/windmill/compare/v1.23.0...v1.24.0) (2022-07-27)
+
+
+### Features
+
+* Add flow input and current step in the prop picker ([#236](https://github.com/windmill-labs/windmill/issues/236)) ([6fbeeae](https://github.com/windmill-labs/windmill/commit/6fbeeae84a207be46490361788dad12918c37c4e))
+* add google login v1 ([fc918a2](https://github.com/windmill-labs/windmill/commit/fc918a24ccf0ad19b81a3ebf630d0f04b56094c8))
+* add schedule settable from pull flows ([caecbfd](https://github.com/windmill-labs/windmill/commit/caecbfd0d9eaadc38372ce7238ed6d3baf9ba6e3))
+* prop picker functional for pull flows ([010acfe](https://github.com/windmill-labs/windmill/commit/010acfe7e365a838078f1a989b54f1539c8bf2e6))
+* skip failures loop ([#258](https://github.com/windmill-labs/windmill/issues/258)) ([de3fe69](https://github.com/windmill-labs/windmill/commit/de3fe699089e2a28aa0032a57a9a03f35646b6ef))
+
+
+### Bug Fixes
+
+* audit logs ([ca4bed3](https://github.com/windmill-labs/windmill/commit/ca4bed34a65440cd790cae9cff19f40df22f92b8))
+* **frontend:** badge google logo for login ([cfec7a9](https://github.com/windmill-labs/windmill/commit/cfec7a97b883dbf83bd9d0707bf015c2aaa4e517))
+* **frontend:** badge needs a little right margin ([c846ed7](https://github.com/windmill-labs/windmill/commit/c846ed76c4102335a5a8aabceaa39d6b7906ef5a))
+* **frontend:** display number field in flows ([a232895](https://github.com/windmill-labs/windmill/commit/a23289563deca70269bd73ec50f324db0b6df791))
+* **frontend:** fork script from hub ([43cacc1](https://github.com/windmill-labs/windmill/commit/43cacc1a66b1e2322c0252c9d1ca954e893aaef8))
+* **frontend:** get refresh token for google services ([2f0d8d5](https://github.com/windmill-labs/windmill/commit/2f0d8d5384fb4eea6a6d5e5e48fd242f8d0c40fa))
+* **frontend:** get refresh token for google services ([8dfe688](https://github.com/windmill-labs/windmill/commit/8dfe688a6a2388cecb1460913a25ab49ec297b1b))
+* **frontend:** get refresh token for google services ([a2c5dc1](https://github.com/windmill-labs/windmill/commit/a2c5dc18a38045cbefc7d4b86d786a3c8fcb3ca8))
+* import from JSON load schemas ([88dd7b0](https://github.com/windmill-labs/windmill/commit/88dd7b0abbd1a0469fc949c8045f61ddc304701d))
+* multiple UI fixes ([a334029](https://github.com/windmill-labs/windmill/commit/a33402978720470530baecf51c2d17ecafd13ab0))
+* multiple UI fixes ([904f0f3](https://github.com/windmill-labs/windmill/commit/904f0f3e69034421d524a66e0c4697ff42d89efe))
+
+## [1.23.0](https://github.com/windmill-labs/windmill/compare/v1.22.0...v1.23.0) (2022-07-25)
+
+
+### Features
+
+* add editor bar to inline scripts of flows ([7a6a2c9](https://github.com/windmill-labs/windmill/commit/7a6a2c982daef9aa80e34aa6cbd4889a3c5ec807))
+* **backend:** do not require visibility on job to see job if in possesion of uuid ([b054229](https://github.com/windmill-labs/windmill/commit/b05422963b27d74de8bb6d3be18538d57a71cfe7))
+* **frontend:** deeper integration with the hub ([bb58eba](https://github.com/windmill-labs/windmill/commit/bb58eba2b521aef67b91cfc23f3ddcc8a001e18f))
+* **frontend:** title everywhere ([38987c6](https://github.com/windmill-labs/windmill/commit/38987c6068c4cc2d9accbc368a67362e74adcabf))
+* hub flows integration ([62777b7](https://github.com/windmill-labs/windmill/commit/62777b7a7888b3456f7f864cbb1acd887b172adc))
+
+
+### Bug Fixes
+
+* display websocket status in flow inline editor ([9e9138e](https://github.com/windmill-labs/windmill/commit/9e9138e4eeaea962dbb149ad4c1450572f025bc5))
+* do not redirect to /user on /user namespace ([d95128e](https://github.com/windmill-labs/windmill/commit/d95128e68190fa6f75871f579de906ce82619524))
+* **oauth2:** add google clients ([bc650b0](https://github.com/windmill-labs/windmill/commit/bc650b0ade1d378f815ee01da480a63ddd4501f1))
+* static is undefined by default instead of being empty '' ([fc65162](https://github.com/windmill-labs/windmill/commit/fc651629c7977b5221dbb101f515766b23af9274))
+
+## [1.22.0](https://github.com/windmill-labs/windmill/compare/v1.21.1...v1.22.0) (2022-07-22)
+
+
+### Features
+
+* add delete schedule ([f6d6934](https://github.com/windmill-labs/windmill/commit/f6d69345841f2ec0d06dc32b59840009982c55f2))
+* **backend:** check of no path conflict between flow and flow's primary schedules ([c346339](https://github.com/windmill-labs/windmill/commit/c34633989e41e215d6183e5c887db68d4cc228d3))
+* dynamic template for script inputs in flow ([3c16621](https://github.com/windmill-labs/windmill/commit/3c16621f6b9c2bee1f2630411bd70d075d247974))
+* import and export flow from JSON ([7862ff4](https://github.com/windmill-labs/windmill/commit/7862ff41e25447d7b34aa261187bb98ed3f3105b))
+* more visual cues about trigger scripts ([36606ab](https://github.com/windmill-labs/windmill/commit/36606ab8b675d01b0d38e2dd883b6e42b0987a6c))
+* more visual cues about trigger scripts ([154c2a9](https://github.com/windmill-labs/windmill/commit/154c2a91ca6a4d60b02a44dda5fa23974594018b))
+* rich rendering of flows ([38ffcfe](https://github.com/windmill-labs/windmill/commit/38ffcfeb292c6e9df0c89a4ef5364cdb8e23ccdd))
+
+
+### Bug Fixes
+
+* **deno-client:** make hack for patching openapi-generator more stable ([08ab4d1](https://github.com/windmill-labs/windmill/commit/08ab4d171a286d94e439a89d97115ad2db8e25d9))
+* export json is converted to pull mode ([666e0f6](https://github.com/windmill-labs/windmill/commit/666e0f68d0dd84fce35e6fe1804c90a3c5125057))
+* export json is converted to pull mode + rd fix ([c7528d4](https://github.com/windmill-labs/windmill/commit/c7528d417f276fbdb96751cda547feec7ac6fbc8))
+* **frontend:** filter script by is_trigger and jobs by is_skipped + path fix ([97292d1](https://github.com/windmill-labs/windmill/commit/97292d18fb7158471f1be6ffbd45a612b09a689f))
+* **frontend:** initFlow also reset schemaStore ([5941467](https://github.com/windmill-labs/windmill/commit/5941467ea19938b4d11b56c6f10f529c87cb52a3))
+* **frontend:** remove unecessary step 1 of flows ([f429074](https://github.com/windmill-labs/windmill/commit/f429074528770f5eaebcf1ce687b6431321e169a))
+* improve tooltip ([4be5d37](https://github.com/windmill-labs/windmill/commit/4be5d37a5441555c83eefbea17e86a5df4946749))
+* improve tooltip ([c84b1c9](https://github.com/windmill-labs/windmill/commit/c84b1c9a8c6a03b9689e3405fa87f3c54016914a))
+* placeholder undefined for arginput ([4d01598](https://github.com/windmill-labs/windmill/commit/4d01598e24fca673b0dc83860e151c21ab403b7a))
+
+## [1.21.1](https://github.com/windmill-labs/windmill/compare/v1.21.0...v1.21.1) (2022-07-19)
+
+
+### Bug Fixes
+
+* **deno-client:** make hack for patching openapi-generator more stable ([2f4df43](https://github.com/windmill-labs/windmill/commit/2f4df43a1a798501449e82767d59f08e9cf95146))
+* **python-client:** sed openapi to avoid generator circular dependency ([49f8050](https://github.com/windmill-labs/windmill/commit/49f8050aaf48c15fb79130a06ce754e285d17dd0))
+
+## [1.21.0](https://github.com/windmill-labs/windmill/compare/v1.20.0...v1.21.0) (2022-07-19)
+
+
+### Features
+
+* add run_wait_result to mimic lambda ability ([6ef3754](https://github.com/windmill-labs/windmill/commit/6ef3754759346b8261934a35bd3bf3983872390f))
+
+
+### Bug Fixes
+
+* **backend:** clear env variables before running script ([98a5959](https://github.com/windmill-labs/windmill/commit/98a5959fcca19c54715e78055cf8881496209ac0))
+* consistent exists/{resource} addition + usage in frontend ([ca66d33](https://github.com/windmill-labs/windmill/commit/ca66d33a4297d2f3a105829650a544f4a89c4615))
+* **frontend:** validate username ([9828e54](https://github.com/windmill-labs/windmill/commit/9828e545e9649bc2ac6af598118ef85580fd80f3))
+* list with is_skipped + deno-client fix ([6939f9d](https://github.com/windmill-labs/windmill/commit/6939f9d76b1579f2932e08df3f67dc293c642fd0))
+
+## [1.20.0](https://github.com/windmill-labs/windmill/compare/v1.19.3...v1.20.0) (2022-07-17)
+
+
+### Features
+
+* trigger scripts and have flows being triggered by checking new external events regularly ([#200](https://github.com/windmill-labs/windmill/issues/200)) ([af23b30](https://github.com/windmill-labs/windmill/commit/af23b30c37b4225d6b927644f9612d4861e2d06c))
+
+
+### Bug Fixes
+
+* flow UI back and forth pull/push fix ([8918eb6](https://github.com/windmill-labs/windmill/commit/8918eb6fdb904e23b5dc340db669f6039ed7abb6))
+* flow UI back and forth pull/push fix ([0973859](https://github.com/windmill-labs/windmill/commit/097385981323d5f88a51eb8df0e1114e8cf62727))
+* **frontend:** chrome columns-2 fix for pull/push ([8272b11](https://github.com/windmill-labs/windmill/commit/8272b1110757ee0ed0cee4a7a6de537fcec83de3))
+* **frontend:** createInlineScript only create trigger script if step = 0 ([bd004cf](https://github.com/windmill-labs/windmill/commit/bd004cff0f5150eb043f5446f5697bea43b1508b))
+* HubPicker pick from trigger scripts when relevant ([7e846c3](https://github.com/windmill-labs/windmill/commit/7e846c32a63d9fe2f46f50f7642918cc34459829))
+
+## [1.19.3](https://github.com/windmill-labs/windmill/compare/v1.19.2...v1.19.3) (2022-07-15)
+
+
+### Bug Fixes
+
+* **deno-client:** do not create resource for createInternalPath ([0967c1b](https://github.com/windmill-labs/windmill/commit/0967c1be65a9803e25f7701850be33121eb44d1b))
+
+## [1.19.2](https://github.com/windmill-labs/windmill/compare/v1.19.1...v1.19.2) (2022-07-15)
+
+
+### Bug Fixes
+
+* **deno-client:** handle text/plain parse ([18e33bb](https://github.com/windmill-labs/windmill/commit/18e33bb40739fd699323f2da87de8c9696c0ef6c))
+
+## [1.19.1](https://github.com/windmill-labs/windmill/compare/v1.19.0...v1.19.1) (2022-07-14)
+
+
+### Bug Fixes
+
+* **backend:** create resource would fail if is_oauth was not set ([cd621a6](https://github.com/windmill-labs/windmill/commit/cd621a6285d2aa0e554434998e931e96110464bd))
+* **deno-client:** handle text/plain serialize ([98968ab](https://github.com/windmill-labs/windmill/commit/98968ab039fea89b7525fe7b852ba3d15dee831e))
+
+## [1.19.0](https://github.com/windmill-labs/windmill/compare/v1.18.0...v1.19.0) (2022-07-14)
+
+
+### Features
+
+* add DISABLE_NSJAIL mode ([1943585](https://github.com/windmill-labs/windmill/commit/19435851de0c18fc876a3bd00f3d9153f2719d9b))
+
+
+### Bug Fixes
+
+* add new ca-certificates folders for nsjail ([2eac1ef](https://github.com/windmill-labs/windmill/commit/2eac1ef363b209bb298dcbe7aafb7282ddd2b87a))
+* **frontend:** add arbitrary scopes to connect an app ([372b14e](https://github.com/windmill-labs/windmill/commit/372b14e158bcb10bcfb07d231afeca5cc780661d))
+* write job arguments to file ([#199](https://github.com/windmill-labs/windmill/issues/199)) ([9a6db75](https://github.com/windmill-labs/windmill/commit/9a6db758c15915f5f0027b1d270d621f91b7ae30))
+
+## [1.18.0](https://github.com/windmill-labs/windmill/compare/v1.17.1...v1.18.0) (2022-07-13)
+
+
+### Features
+
+* account part II, handle refresh tokens, clarify oauth UI ([#196](https://github.com/windmill-labs/windmill/issues/196)) ([8403fbb](https://github.com/windmill-labs/windmill/commit/8403fbbc02076bb37dc82b2d26685957b13d036b))
+
+
+### Bug Fixes
+
+* **frontend:** fix path group refresh & create variable path reset ([6a341f5](https://github.com/windmill-labs/windmill/commit/6a341f5dc343df3df6491f8026e87632979faace))
+
+## [1.17.1](https://github.com/windmill-labs/windmill/compare/v1.17.0...v1.17.1) (2022-07-08)
+
+
+### Bug Fixes
+
+* **backend:** set error content-type to text ([cf2dfd7](https://github.com/windmill-labs/windmill/commit/cf2dfd7fe74956d68bdc26dc47557ea6a0ed1ce4))
+* **deno-client:** fix stringify ([5b89abe](https://github.com/windmill-labs/windmill/commit/5b89abe28283238a282da8920580a72f25e5a360))
+* **frontend:** change lsp behavior ([d6e0817](https://github.com/windmill-labs/windmill/commit/d6e0817dc4fe54efd9346698c0ccb39057921d9b))
+* **frontend:** connect an app resource creation ([e400dcc](https://github.com/windmill-labs/windmill/commit/e400dccedd88e3f5e3a9b0ec52fc9883d60c959b))
+* **frontend:** connect an app resource creation ([68c5318](https://github.com/windmill-labs/windmill/commit/68c5318d16c85a01822570c113a4f33c539dc8bf))
+* **frontend:** current hash link ([22eef8a](https://github.com/windmill-labs/windmill/commit/22eef8afab9143bb5b110db8c76e024604106051))
+* **frontend:** fix sendRequest ([5da9819](https://github.com/windmill-labs/windmill/commit/5da9819ca5ce15ef4de9cf4a84affbd581383483))
+* **frontend:** reload editor when language changes for in-flow editor ([72c7890](https://github.com/windmill-labs/windmill/commit/72c7890427736eeeb9a872bf0efd1acc906efd63))
+* **frontend:** sveltekit prerender enabled -> default ([635873a](https://github.com/windmill-labs/windmill/commit/635873a96a586ad8e936526f4f4ebf679519e7fc))
+* in-flow script editor fixes ([466f6b3](https://github.com/windmill-labs/windmill/commit/466f6b339acf70351814c32b8f31d80b8ff1c1b5))
+* in-flow script editor fixes ([5853dfd](https://github.com/windmill-labs/windmill/commit/5853dfd85dca3c80b0edfb58b2866948af8011d5))
+* remove unnecessary v8 snapshot ([d3904fd](https://github.com/windmill-labs/windmill/commit/d3904fd3ebde3a200ccc157a8532dfe1435ae16d))
+
+## [1.17.0](https://github.com/windmill-labs/windmill/compare/v1.16.1...v1.17.0) (2022-07-05)
+
+
+### Features
+
+* in-flow editor mvp ([330b373](https://github.com/windmill-labs/windmill/commit/330b373c24f21b4d9a9b2903e8f1c60ee784ea89))
+
+## [1.16.1](https://github.com/windmill-labs/windmill/compare/v1.16.0...v1.16.1) (2022-07-05)
+
+
+### Bug Fixes
+
+* bump all backend deps by breaking cycling through not using oauth2 ([e4a6378](https://github.com/windmill-labs/windmill/commit/e4a637860133e78cb1675173ccf3ff45e4b08c09))
+* oauth logins used incorrect scope ([1dcba67](https://github.com/windmill-labs/windmill/commit/1dcba67a1f607faabcdfa6f7e94d280c66dd6470))
+* trace errors body ([d092c62](https://github.com/windmill-labs/windmill/commit/d092c622c4efadb1e2799f7dbbe03f825f2b364d))
+
+
+## [1.16.0](https://github.com/windmill-labs/windmill/compare/v1.15.1...v1.16.0) (2022-07-02)
+
+
+### Features
+
+* OAuth "Connect an App" ([#155](https://github.com/windmill-labs/windmill/issues/155)) ([3636866](https://github.com/windmill-labs/windmill/commit/3636866dda8b2e14d61c99a76f0a4e5fa6a37123))
+
+
+### Bug Fixes
+
+* add gitlab to connects ([d4e7c9e](https://github.com/windmill-labs/windmill/commit/d4e7c9e171cd02a7aa0846b43c127720260600b5))
+* diverse frontend fixes
+
+## [1.15.1](https://github.com/windmill-labs/windmill/compare/v1.15.0...v1.15.1) (2022-06-29)
+
+
+### Bug Fixes
+
+* databaseUrlFromResource uses proper database field ([6954580](https://github.com/windmill-labs/windmill/commit/69545808012fa4f5080ec58cf3dff2961a327117))
+
+## [1.15.0](https://github.com/windmill-labs/windmill/compare/v1.14.6...v1.15.0) (2022-06-29)
+
+
+### Features
+
+* Flows Property picker component + Dynamic type inference ([#129](https://github.com/windmill-labs/windmill/issues/129)) ([44b4acf](https://github.com/windmill-labs/windmill/commit/44b4acf4bcfa0c372a9938a9b97d31cceedd9ad9))
+
+## [1.14.6](https://github.com/windmill-labs/windmill/compare/v1.14.5...v1.14.6) (2022-06-27)
+
+
+### Bug Fixes
+
+* add databaseUrlFromResource to deno ([2659e9d](https://github.com/windmill-labs/windmill/commit/2659e9d62b88c2127c969becbc3a61ed2f118069))
+
+## [1.14.5](https://github.com/windmill-labs/windmill/compare/v1.14.4...v1.14.5) (2022-06-27)
+
+
+### Bug Fixes
+
+* index.ts -> mod.ts ([d41913a](https://github.com/windmill-labs/windmill/commit/d41913a440b2034de59437488edc85e38c956d5f))
+* insert getResource proper parenthesis ([e07b5d4](https://github.com/windmill-labs/windmill/commit/e07b5d4f30ea79a99caac4fb63a9ab1f17eaaf74))
+
+## [1.14.4](https://github.com/windmill-labs/windmill/compare/v1.14.3...v1.14.4) (2022-06-27)
+
+
+### Bug Fixes
+
+* windmill deno package index.ts -> mod.ts ([8c0acac](https://github.com/windmill-labs/windmill/commit/8c0acac212d742acee8b7ff0cf6b93cce4187c19))
+
+## [1.14.3](https://github.com/windmill-labs/windmill/compare/v1.14.2...v1.14.3) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v3 ([31445d7](https://github.com/windmill-labs/windmill/commit/31445d7182a910eab9d699760f2a86ca23d556a4))
+* internal state for script triggers v3 ([22c6347](https://github.com/windmill-labs/windmill/commit/22c6347d8a74d94dc18109390ff5c347a2732823))
+* internal state for script triggers v4 ([63a7401](https://github.com/windmill-labs/windmill/commit/63a7401f248cc37951bbea4dcaedaa6497d6f0b1))
+
+## [1.14.2](https://github.com/windmill-labs/windmill/compare/v1.14.1...v1.14.2) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v2 ([f9eedc3](https://github.com/windmill-labs/windmill/commit/f9eedc31ed6e5d7e0a8a26633cca9965ac3b6a05))
+
+## [1.14.1](https://github.com/windmill-labs/windmill/compare/v1.14.0...v1.14.1) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v1 ([6321311](https://github.com/windmill-labs/windmill/commit/6321311112dfa3ef09447f41847b248c0e0dcb46))
+
+## [1.14.0](https://github.com/windmill-labs/windmill/compare/v1.13.0...v1.14.0) (2022-06-27)
+
+
+### Features
+
+* add tesseract bin to worker image ([6de9697](https://github.com/windmill-labs/windmill/commit/6de9697d955a06cfb9c64fdb501b4dfa1bb597ad))
+* deno run with --unstable ([4947661](https://github.com/windmill-labs/windmill/commit/4947661b1d91867c022bb8a10a4be3e91f69352c))
+* internal state for script triggers mvp ([dcdb989](https://github.com/windmill-labs/windmill/commit/dcdb989adb8350974289a0c8d2239b245a6e0d41))
+
+
+### Bug Fixes
+
+* change default per page to 100 ([fdf95a0](https://github.com/windmill-labs/windmill/commit/fdf95a065e83d733ab6a0f02edb4af16c0a1dfb9))
+* deno exit after result logging ([6c622bc](https://github.com/windmill-labs/windmill/commit/6c622bcc32473361e1f7cb1ea7b0b508929bc1b8))
+* improve error handling ([f98f642](https://github.com/windmill-labs/windmill/commit/f98f6429c1e646c0a836f2f73a03a803aa655583))
+* improve error handling ([2efaf21](https://github.com/windmill-labs/windmill/commit/2efaf2191551c1406618c6d60bd37ca6eff84560))
+* schemaPicker does not display editor by default ([fc0c38f](https://github.com/windmill-labs/windmill/commit/fc0c38ffad18a9ceda44cb8406736c14ba4eb4c2))
+* smart assistant reload ([bb946ed](https://github.com/windmill-labs/windmill/commit/bb946ed5519f59adc559d6959c56e61403389c9d))
+
+## [1.13.0](https://github.com/windmill-labs/windmill/compare/v1.12.0...v1.13.0) (2022-06-22)
+
+
+### Features
+
+* better type narrowing for list and array types ([276319d](https://github.com/windmill-labs/windmill/commit/276319d99240dbca5bcc74a1142d99ca823c4da2))
+
+
+### Bug Fixes
+
+* fix webhook path for flows ([906f740](https://github.com/windmill-labs/windmill/commit/906f740a0ddce26743e4669af7a101613131a17c))
+* make email constraint case insensitive ([6dc90a3](https://github.com/windmill-labs/windmill/commit/6dc90a390643fcf6116289596ca1c3149d326797))
+
+## [1.12.0](https://github.com/windmill-labs/windmill/compare/v1.11.0...v1.12.0) (2022-06-14)
+
+
+### Bug Fixes
+
+* more flexible ResourceType MainArgSignature parser ([359ef15](https://github.com/windmill-labs/windmill/commit/359ef15fa2a9024507a71f2c656373925fba3ebe))
+* rename ResourceType -> Resource ([28b5671](https://github.com/windmill-labs/windmill/commit/28b56714023ea69a20f003e08f6c40de64202ac5))
+
+## [1.11.0](https://github.com/windmill-labs/windmill/compare/v1.10.1...v1.11.0) (2022-06-13)
+
+
+### Features
+
+* add DISABLE_NUSER for older kernels ([cce46f9](https://github.com/windmill-labs/windmill/commit/cce46f94404ac5c10407e430fff8cdec3bd7fb2d))
+* add ResourceType<'name'> as deno signature arg type ([f1ee5f3](https://github.com/windmill-labs/windmill/commit/f1ee5f3130cb7b753ccc3ee62169c5e4a8ef7b8b))
+
+
+### Bug Fixes
+
+* force c_ prefix for adding resource type ([9f235c4](https://github.com/windmill-labs/windmill/commit/9f235c404ed62b54a73451b9f9dbddd8f013120d))
+* **frontend:** loadItems not called in script picker ([a59b927](https://github.com/windmill-labs/windmill/commit/a59b92706b24a07cc14288620a9bcdb9402bd134))
+
+## [1.10.1](https://github.com/windmill-labs/windmill/compare/v1.10.0...v1.10.1) (2022-06-12)
+
+
+### Bug Fixes
+
+* python-client verify ssl ([295e28f](https://github.com/windmill-labs/windmill/commit/295e28fd43ef07b739d2c7c85b0ae6819f7d7434))
+
+## [1.10.0](https://github.com/windmill-labs/windmill/compare/v1.9.0...v1.10.0) (2022-06-11)
+
+
+### Features
+
+* alpha hub integration + frontend user store fixes + script client base_url fix ([1a61d50](https://github.com/windmill-labs/windmill/commit/1a61d50076b295fe97e48c2a621dff30802152b1))
+
+## [1.9.0](https://github.com/windmill-labs/windmill/compare/v1.8.6...v1.9.0) (2022-06-05)
+
+
+### Features
+
+* update postgres 13->14 in docker-compose ([479a12f](https://github.com/windmill-labs/windmill/commit/479a12f33ca26bfd1b67bcdd24a64ca26cc6bebe))
+
+
+### Bug Fixes
+
+* remove annoying transitions for scripts and flows ([f2348b5](https://github.com/windmill-labs/windmill/commit/f2348b5526bb8197519685cb57049f74c6f3a11d))
+
+### [1.8.6](https://github.com/windmill-labs/windmill/compare/v1.8.5...v1.8.6) (2022-05-18)
+
+
+### Bug Fixes
+
+* re-release ([d31cd3c](https://github.com/windmill-labs/windmill/commit/d31cd3c52c1b46e821da261f22d0aec872b61fb2))
+
+### [1.8.5](https://github.com/windmill-labs/windmill/compare/v1.8.4...v1.8.5) (2022-05-18)
+
+
+### Bug Fixes
+
+* language field broke flow too ([33fed8e](https://github.com/windmill-labs/windmill/commit/33fed8e04d3abbde371535ecb6e7ba15d103db92))
+
+### [1.8.4](https://github.com/windmill-labs/windmill/compare/v1.8.3...v1.8.4) (2022-05-18)
+
+
+### Bug Fixes
+
+* scripts run was broken due to 1.7 and 1.8 changes. This fix it ([7564d2c](https://github.com/windmill-labs/windmill/commit/7564d2cb1e7f600ede22f333a02a537df381d829))
+
+### [1.8.3](https://github.com/windmill-labs/windmill/compare/v1.8.2...v1.8.3) (2022-05-18)
+
+
+### Bug Fixes
+
+* clean exported deno-client api ([605c2b4](https://github.com/windmill-labs/windmill/commit/605c2b4d11bf072332a38f0c3e24cf6cc9ec7e65))
+
+### [1.8.2](https://github.com/windmill-labs/windmill/compare/v1.8.1...v1.8.2) (2022-05-18)
+
+
+### Bug Fixes
+
+* deno client ([563ba3e](https://github.com/windmill-labs/windmill/commit/563ba3e7f763279a93f619933ac35a1dec3f727a))
+* deno lsp client ([3eed59f](https://github.com/windmill-labs/windmill/commit/3eed59fcb1b172ab13f65c9a0caa0545f5ed91da))
+* deno lsp uses wss instead of ws ([865d728](https://github.com/windmill-labs/windmill/commit/865d728224bed55fe4a2c1905ff2b8c15f4bbe17))
+* starting deno script is now async ([7365a8e](https://github.com/windmill-labs/windmill/commit/7365a8e87bdb1f879eb92125a9e6378a1636637e))
+
+### [1.8.1](https://github.com/windmill-labs/windmill/compare/v1.8.0...v1.8.1) (2022-05-17)
+
+
+### Bug Fixes
+
+* frontend dependencies update ([f793bc4](https://github.com/windmill-labs/windmill/commit/f793bc46d98349a5fea56c7911b6e0720b2b117c))
+
+## [1.8.0](https://github.com/windmill-labs/windmill/compare/v1.7.0...v1.8.0) (2022-05-17)
+
+
+### Features
+
+* Typescript support for scripts (alpha) ([2e1d430](https://github.com/windmill-labs/windmill/commit/2e1d43033f3ad6dbe86338b7a41da7b1120a5ffc))
+
+## [1.7.0](https://github.com/windmill-labs/windmill/compare/v1.6.1...v1.7.0) (2022-05-14)
+
+
+### Features
+
+* self host github oauth ([#46](https://github.com/windmill-labs/windmill/issues/46)) ([5b413d7](https://github.com/windmill-labs/windmill/commit/5b413d7e045d09dc5c5916cb22d82438ec6c92ad))
+
+
+### Bug Fixes
+
+* better error message when saving script ([02c8bea](https://github.com/windmill-labs/windmill/commit/02c8bea0840e492c31ccb8ddd1e5ae9676a534b1))
+
+### [1.6.1](https://github.com/windmill-labs/windmill/compare/v1.6.0...v1.6.1) (2022-05-10)
+
+
+### Bug Fixes
+
+* also store and display "started at" for completed jobs ([#33](https://github.com/windmill-labs/windmill/issues/33)) ([2c28031](https://github.com/windmill-labs/windmill/commit/2c28031e44453740ad8c4b7e3c248173eab34b9c))
+
+## 1.6.0 (2022-05-10)
+
+### Features
+
+* superadmin settings ([7a51f84](https://www.github.com/windmill-labs/windmill/commit/7a51f842f01e17c4d230c060fa0de558553ad3ed))
+* user settings is now at workspace level ([a130806](https://www.github.com/windmill-labs/windmill/commit/a130806e1929267ee40ca443e3dac6e1a5d80da3))
+
+
+### Bug Fixes
+
+* display more than default 30 workspaces as superadmin ([55b5695](https://www.github.com/windmill-labs/windmill/commit/55b5695673912ffe040d3011c020b1002b4e3268))
+
 ## [1.5.0](https://www.github.com/windmill-labs/windmill/v1.5.0) (2022-05-02)

CLA.md

@@ -0,0 +1,145 @@
+## Contributor Agreement
+
+## Individual Contributor Non-Exclusive License Agreement
+
+Thank you for your interest in contributing to Windmill Labs, Inc's Windmill
+("We" or "Us").
+
+The purpose of this contributor agreement ("Agreement") is to clarify and
+document the rights granted by contributors to Us.
+
+### 1\. Definitions
+
+**"You"** means the individual Copyright owner who Submits a Contribution to Us.
+
+**"Legal Entity"** means an entity that is not a natural person.
+
+**"Affiliate"** means any other Legal Entity that controls, is controlled by, or
+under common control with that Legal Entity. For the purposes of this
+definition, "control" means (i) the power, direct or indirect, to cause the
+direction or management of such Legal Entity, whether by contract or otherwise,
+(ii) ownership of fifty percent (50%) or more of the outstanding shares or
+securities that vote to elect the management or other persons who direct such
+Legal Entity or (iii) beneficial ownership of such entity.
+
+**"Contribution"** means any original work of authorship, including any original
+modifications or additions to an existing work of authorship, Submitted by You
+to Us, in which You own the Copyright.
+
+**"Copyright"** means all rights protecting works of authorship, including
+copyright, moral and neighboring rights, as appropriate, for the full term of
+their existence.
+
+**"Material"** means the software or documentation made available by Us to third
+parties.
+
+**"Submit"** means any act by which a Contribution is transferred to Us by You
+by means of tangible or intangible media, including but not limited to
+electronic mailing lists, source code control systems, and issue tracking
+systems that are managed by, or on behalf of, Us, but excluding any transfer
+that is conspicuously marked or otherwise designated in writing by You as "Not a
+Contribution."
+
+**"Documentation"** means any non-software portion of a Contribution.
+
+### 2\. License grant
+
+#### 2.1 Copyright license to Us
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Us a
+worldwide, royalty-free, NON-exclusive, perpetual and irrevocable (except as
+stated in Section 8.2) license, with the right to transfer an unlimited number
+of non-exclusive licenses or to grant sublicenses to third parties, under the
+Copyright covering the Contribution to use the Contribution by all means,
+including, but not limited to:
+
+- publish the Contribution,
+- modify the Contribution,
+- prepare derivative works based upon or containing the Contribution and/or to
+  combine the Contribution with other Materials,
+- reproduce the Contribution in original or modified form,
+- distribute, to make the Contribution available to the public, display and
+  publicly perform the Contribution in original or modified form.
+
+#### 2.2 Moral rights
+
+Moral Rights remain unaffected to the extent they are recognized and not
+waivable by applicable law. Notwithstanding, You may add your name to the
+attribution mechanism customary used in the Materials you Contribute to, such as
+the header of the source code files of Your Contribution, and We will respect
+this attribution when using Your Contribution.
+
+### 3\. Patents
+
+#### 3.1 Patent license
+
+Subject to the terms and conditions of this Agreement You hereby grant to Us and
+to recipients of Materials distributed by Us a worldwide, royalty-free,
+non-exclusive, perpetual and irrevocable (except as stated in Section 3.2)
+patent license, with the right to transfer an unlimited number of non-exclusive
+licenses or to grant sublicenses to third parties, to make, have made, use,
+sell, offer for sale, import and otherwise transfer the Contribution and the
+Contribution in combination with any Material (and portions of such
+combination). This license applies to all patents owned or controlled by You,
+whether already acquired or hereafter acquired, that would be infringed by
+making, having made, using, selling, offering for sale, importing or otherwise
+transferring of Your Contribution(s) alone or by combination of Your
+Contribution(s) with any Material.
+
+### 4. Disclaimer
+
+THE CONTRIBUTION IS PROVIDED "AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED
+WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF SATISFACTORY
+QUALITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY
+DISCLAIMED BY YOU TO US AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES
+CANNOT BE DISCLAIMED, SUCH WARRANTY IS LIMITED IN DURATION AND EXTENT TO THE
+MINIMUM PERIOD AND EXTENT PERMITTED BY LAW.
+
+### 5. Consequential damage waiver
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR WE BE
+LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR
+OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+### 6. Approximation of disclaimer and damage waiver
+
+IF THE DISCLAIMER AND DAMAGE WAIVER MENTIONED IN SECTION 4. AND SECTION 5.
+CANNOT BE GIVEN LEGAL EFFECT UNDER APPLICABLE LOCAL LAW, REVIEWING COURTS SHALL
+APPLY LOCAL LAW THAT MOST CLOSELY APPROXIMATES AN ABSOLUTE WAIVER OF ALL CIVIL
+OR CONTRACTUAL LIABILITY IN CONNECTION WITH THE CONTRIBUTION.
+
+### 7. Term
+
+7.1 This Agreement shall come into effect upon Your acceptance of the terms and
+conditions.
+
+7.3 In the event of a termination of this Agreement Sections 4, 5, 6, 7 and 8
+shall survive such termination and shall remain in full force thereafter. For
+the avoidance of doubt, Free and Open Source Software (sub)licenses that have
+already been granted for Contributions at the date of the termination shall
+remain in full force after the termination of this Agreement.
+
+### 8 Miscellaneous
+
+8.1 This Agreement and all disputes, claims, actions, suits or other proceedings
+arising out of this agreement or relating in any way to it shall be governed by
+the laws of France excluding its private international law provisions.
+
+8.2 This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+8.3 In case of Your death, this agreement shall continue with Your heirs. In
+case of more than one heir, all heirs must exercise their rights through a
+commonly authorized person.
+
+8.4 If any provision of this Agreement is found void and unenforceable, such
+provision will be replaced to the extent possible with a provision that comes
+closest to the meaning of the original provision and that is enforceable. The
+terms and conditions set forth in this Agreement shall apply notwithstanding any
+failure of essential purpose of this Agreement or any limited remedy to the
+maximum extent possible under law.
+
+8.5 You agree to notify Us of any facts or circumstances of which you become
+aware that would make this Agreement inaccurate in any respect.

Caddyfile

@@ -1,4 +1,5 @@
-{$SITE_URL} {  
-  bind {$ADDRESS}
-  reverse_proxy /* server:8000
+{$BASE_URL} {
+        bind {$ADDRESS}
+        reverse_proxy /ws/* http://lsp:3001
+        reverse_proxy /* http://windmill:8000
 }

Dockerfile

@@ -19,7 +19,7 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
 RUN make
 
-FROM mhart/alpine-node:14 as frontend
+FROM node:18-alpine as frontend
 
 # install dependencies
 WORKDIR /frontend
@@ -30,8 +30,11 @@ RUN npm ci
 COPY frontend .
 RUN mkdir /backend
 COPY /backend/openapi.yaml /backend/openapi.yaml
+COPY /openflow.openapi.yaml /openflow.openapi.yaml
 RUN npm run generate-backend-client
+ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
+RUN npm run check
 
 FROM rust:slim-buster as builder
 
@@ -46,7 +49,7 @@ COPY ./backend/.cargo/ .cargo/
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl
+    curl lld
 
 ENV CARGO_INCREMENTAL=1
 
@@ -56,11 +59,11 @@ RUN rm src/*.rs
 RUN rm ./target/release/deps/windmill*
 ENV SQLX_OFFLINE=true
 
-ADD ./backend ./
-ADD ./nsjail /nsjail
+COPY ./backend ./
+COPY ./nsjail /nsjail
 
-COPY --from=1 /frontend /frontend
-ADD .git/ .git/
+COPY --from=frontend /frontend /frontend
+COPY .git/ .git/
 
 RUN cargo build --release
 
@@ -69,11 +72,11 @@ FROM debian:buster-slim
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 python3 python3-pip \
+    && apt-get install -y ca-certificates tzdata libpq5 \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev \
+    libv8-dev tesseract-ocr \
     && rm -rf /var/lib/apt/lists/*
 
 ENV TZ=Etc/UTC
@@ -84,12 +87,16 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
     && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
     && make -j 4 && make install
 
-RUN python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install nltk
+RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -2,7 +2,7 @@
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
 
-Every file is under copyright (c) Ruben Fiszel 2021 unless otherwise specified.
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 

LICENSE-APACHE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2021 Ruben Fiszel
+   Copyright 2022 Windmill Labs, Inc
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -1,6 +1,4 @@
-Ruben Fiszel
-
-Copyright (c) 2021  Ruben Fiszel
+Copyright (c) 2022  Windmill Labs, Inc
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 or the GNU Affero General Public License. Please see

README.md

@@ -1,10 +1,17 @@
 <p align="center">
-  <a href="https://alpha.windmill.dev"><img src="./windmill.svg" alt="windmill.dev"></a>
+  <a href="https://app.windmill.dev"><img src="./imgs/windmill.svg" alt="windmill.dev"></a>
 </p>
 <p align="center">
-    <em>Windmill.dev is an OSS developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
+    <em>.</em>
+</p>
+<p align=center>
+Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger flows and scripts as internal apps. Convert code to no-code modules and, if the auto-generated UI is not sufficient, use it solely as an highly scalable backend layer. Add automation to your product or build your own no-code tool and delegate the core layer to Windmill
+.
 </p>
 <p align="center">
+<a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
+    <img src="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml/badge.svg" alt="Docker Image CI">
+</a>
 <a href="https://pypi.org/project/wmill" target="_blank">
     <img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
@@ -15,81 +22,250 @@
 
 ---
 
-**Join the alpha (personal workspaces are free forever)**:
-<https://alpha.windmill.dev>
+**Join the beta (personal workspaces are free forever)**:
+<https://app.windmill.dev>
 
 **Documentation**: <https://docs.windmill.dev>
 
 **Discord**: <https://discord.gg/V7PM2YHsPB>
 
-**We are hiring**: Software Engineers, DevOps, Solutions Engineers, Growth:
-<https://docs.windmill.dev/hiring>
+**Hub**: <https://hub.windmill.dev>
+
+**Contributor's guide**: <https://docs.windmill.dev/docs/contributors_guide>
+
+**Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
+
+**[Self-host instruction](#how-to-self-host)**
 
 You can show your support for the project by starring this repo.
 
 ---
 
+Windmill Labs offers commercial licenses and support to convert your existing
+automation and help you scale it in production. If interested, contact
+ruben@windmill.dev (founder of Windmill).
+
+---
+
 # Windmill
 
 <p align="center">
-<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but the API might change,
-especially concerning flows.
+<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
 </p>
 
-![Windmill](./windmill.webp)
+![Windmill Screenshot](./imgs/windmill.png)
 
 Windmill is <b>fully open-sourced</b>:
 
-- `community/` and `python-client/` are Apache 2.0
+- `community/`, `python-client/` and `deno-client/` are Apache 2.0
 - backend, frontend and everything else under AGPLv3.
 
+## What is the general idea behind Windmill
+
+1. Define a minimal and generic script in Python or Typescript that solve a
+   specific task. Here sending an email with SMTP. The code can be defined in
+   the provided Web IDE or synchronized with your own github repo:
+   ![Step 1](./imgs/python-script.png)
+
+2. Your scripts parameters are automatically parsed and generate a frontend. You
+   can narrow down the types during task definition to specify regex for string,
+   an enum or a specific format for objects. Each script correspond to an app by
+   itself: ![Step 2](./imgs/arguments.png)
+
+3. Make it flow! You can chain your scripts or scripts made by the community
+   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
+   between Windmill and the hub to make it easy to build flows from a soon-to-be
+   exhaustive library of generic modules. In flows, one can pipe output to input
+   using "Dynamic" expressions that are just plain Javascript underneath. Flows
+   can contain for-loops, branching (coming soon). As such and coupled with
+   inputs being able to refer to any step's output, they are actual DAG rather
+   than just linear sequences. They are backed by an open JSON spec we call
+   [OpenFlow](https://docs.windmill.dev/docs/openflow)
+   ![Step 3](./imgs/flow.png)
+
+Both scripts and flows are not restricted to be triggered by the UI. They can be
+triggered by a schedule, watch for changes (using
+[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
+triggered through API with either an async or sync webhook. The latter kind of
+endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
+central place to host, build and run all of your integrations, automation and
+internal apps. We include credentials management and OAuth integration, groups
+and much more!
+
 ## Layout
 
 - `backend/`: The whole Rust backend
-- `frontend`: The whole Svelte fronten
-- `community/`: Scripts and resource types created and curated by the community,
-  included in every workspace
+- `frontend`: The whole Svelte frontend
+- `community/`: Scripts and resource types included in every workspace. It is
+  useful for Python scripts since the [WindmillHub](https://hub.windmill.dev)
+  only allow deno scripts and for sharing resource types that will be included
+  in every workspace.
 - `lsp/`: The lsp asssistant for the monaco editor
 - `nsjail/`: The nsjail configuration files for sandboxing of the scripts'
   execution
 - `python-client/`: The wmill python client used within scripts to interact with
   the windmill platform
+- `deno-client/`: The wmill deno client used within scripts to interact with the
+  windmill platform
 
 ## Stack
 
-- postgres as the database
-- backend in Rust with the follwing highly-available and horizontally scalable
+- Postgres as the database
+- backend in Rust with the following highly-available and horizontally scalable
   architecture:
   - stateless API backend
-  - workers that pull jobs from a queue
-- frontend in svelte
-- scripts executions are sandboxed using google's nsjail
-- javascript runtime is deno_core rust library (which itself uses the rusty_v8
-  and hence V8 underneath)
+  - workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
+    Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
+    interested )
+- frontend in Svelte
+- scripts executions are sandboxed using google's
+  [nsjail](https://github.com/google/nsjail)
+- javascript runtime is the
+  [deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses
+  the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
 
+## Security
+
+### Sandboxing and workload isolation
+
+Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
+sandboxing. It is production multi-tenant grade secure. Do not take our word for
+it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
+
+### Secrets, credentials and sensitive values
+
+There is one encryption key per workspace to encrypt the credentials and secrets
+stored in Windmill's K/V store.
+
+In addition, we strongly recommend that you encrypt the whole Postgres database.
+That is what we do at <https://app.windmill.dev>.
+
+## Performance
+
+The performances are great, as long as you do not exceed the parrallelism of the
+workers, we are
+[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
+
 ## Architecture
 
-A detailed section about Windmill architecture is coming soon
+<p align="center">
 
-### Development stack
+### Big-picture Architecture
 
-- caddy is the reverse proxy used for local development, see frontend's
-  Caddyfile and CaddyfileRemote
+<img src="./imgs/diagram.svg">
+
+### Technical Architecture
+
+<img src="./imgs/architecture.svg">
+
+</p>
 
 ## How to self-host
 
-Complete instructions coming soon
+We only provide docker-compose setup here. For more advanced setups, like
+compiling from source or using without a postgres super user, see
+[documentation](https://docs.windmill.dev/docs/how-tos/self_host)
+
+### Docker compose
+
+`docker compose up` with the following docker-compose is sufficient:
+<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
+
+For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
+nsjail will not be able to launch the isolated scripts.
+
+To disable nsjail altogether, set `DISABLE_NSJAIL=true`.
+
+The default super-admin user is: admin@windmill.dev / changeme
+
+From there, you can create other users (do not forget to change the password!)
+
+### Commercial license
+
+To self-host Windmill, you must respect the terms of the AGPLv3 license which
+you do not need to worry about for personal uses. For business uses, you should
+be fine if you do not re-expose it in any way Windmill to your users and are
+comfortable with AGPLv3.
+
+To re-expose any Windmill parts to your users as a feature of your product, or
+to build a feature on top of Windmill, to comply with AGPLv3 your product must
+be AGPLv3 or you must get a commercial license. Contact us at
+<license@windmill.dev> if you have any doubts.
+
+In addition, a commercial license grants you a dedicated engineer to transition
+your current infrastructure to Windmill, support with tight SLA, audit logs
+export features, SSO, unlimited users creation, advanced permissioning features
+such as groups and the ability to create more than one workspace.
+
+### OAuth for self-hosting (very optional)
+
+To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
+the following format:
+
+```json
+{
+  "<client>": {
+    "id": "<CLIENT_ID>",
+    "secret": "<CLIENT_SECRET>"
+  }
+}
+```
+
+and mount it at `/usr/src/app/oauth.json`.
+
+[The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
+
+To add more "connect an app" OAuth clients to the Windmill project, read the
+[Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
+welcome contributions!
+
+### Resource types
+
+You will also want to import all the approved resource types from
+[WindmillHub](https://hub.windmill.dev). There is no automatic way to do this
+automatically currently, but it will be possible using a command with the
+upcoming CLI tool.
+
+## Run a local dev setup
+
+### only Frontend
+
+This will use the backend of <https://app.windmill.dev> but your own frontend
+with hot-code reloading.
+
+1. Install [caddy](https://caddyserver.com)
+2. Go to `frontend/`:
+   1. `npm run install`, `npm run generate-backend-client` then `npm run dev`
+   2. In another shell `sudo caddy run --config CaddyfileRemote`
+3. Et voilà, windmill should be available at `http://localhost/`
+
+### Backend + Frontend
+
+See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all running options.
+
+1. Create a Postgres Database for Windmill and create an admin role inside your
+   Postgres setup.
+2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
+   your PATH
+3. Install deno and python3, have the bins at `/usr/bin/deno` and
+   `/usr/local/bin/python3`
+4. Install [caddy](https://caddyserver.com)
+5. Install the [lld linker](https://lld.llvm.org/)
+6. Go to `backend/`:
+   `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
+7. Go to `frontend/`:
+   1. `npm run install`, `npm run generate-backend-client` then `npm run dev`
+   2. In another shell `sudo caddy run --config Caddyfile`
+8. Et voilà, windmill should be available at `http://localhost/`
+
+## Contributors
+
+<a href="https://github.com/windmill-labs/windmill/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=windmill-labs/windmill" />
+</a>
 
 ## Copyright
 
-2021 [Ruben Fiszel](https://github.com/rubenfiszel)
-
-### Acknowledgement
-
-This project is inspired from a previous project called
-[Delightool](https://github.com/windmill-labs/delightool-legacy) which was also
-led by [Ruben](https://github.com/rubenfiszel) and with large contribution on
-the frontend from [Malo Marrec](https://github.com/malomarrec) who gave his
-blessing to Windmill.
+Windmill Labs, Inc 2022

backend/.cargo/config

@@ -1,3 +1,3 @@
 [build]
-rustflags = ["--cfg", "tokio_unstable"]
+rustflags = ["--cfg", "tokio_unstable", "-C", "link-arg=-fuse-ld=lld"]
 incremental = true

backend/.gitattributes

@@ -0,0 +1 @@
+sqlx-data.json -diff

backend/.gitignore

@@ -1,2 +1,3 @@
 target/
 .env
+oauth.json

backend/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "windmill"
-version = "1.5.0"
-authors = ["Ruben Fiszel <ruben@rubenfiszel.com>"]
+version = "1.34.0"
+authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
 [build-dependencies]
@@ -17,13 +17,15 @@ tower-http = { version = "^0", features = ["trace"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
-uuid = { version = "^0", features = ["serde", "v4"] }
+uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"]}
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"]}
 console-subscriber = "^0"
+prometheus = { version = "^0", default-features = false }
+phf = { version = "0.11", features = ["macros"] }
 
 rust-embed = "^6"
 mime_guess = "^2"
@@ -31,32 +33,33 @@ hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
 retainer = "^0"
-rand = "^0.8.4"
-rand_core = { version = "^0.6.3", features = ["std"] }
+rand = "^0"
+rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
 rustpython-parser = "^0"
 cron = "^0"
-external-ip = "^4"
-lettre = { version = "^0.10.0-rc.4", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
+lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
 urlencoding = "^2"
-oauth2 = "^4"
 url = "^2"
+async-oauth2 = "^0"
 reqwest = { version = "^0", features = ["json"] }
-time = "0.3.7"
+time = "^0"
 slack-http-verifier = "^0"
 serde_urlencoded = "^0"
 tokio-tar = "^0"
 tempfile = "^3"
-tokio-util = { version = "0.7.0", features = ["io"] }
+tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 deno_core = "^0"
-indexmap = "~1.6.2"
 async-recursion = "^1"
+swc_common = "^0"
+swc_ecma_parser = "^0"
+swc_ecma_ast = "^0"
 
 sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
 dotenv = "^0"
-ulid = { version = "^0", features = ["uuid"] }
+ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"

backend/NOTICE

@@ -1,5 +1,5 @@
-Ruben Fiszel
+Windmill Labs, Inc
 
-Copyright (c) 2021  Ruben Fiszel
+Copyright (c) 2021  Windmill Labs, Inc
 
 Source code in this directory is licensed the GNU Affero General Public License. 

backend/build.rs

@@ -1,17 +0,0 @@
-use std::fs::File;
-use std::io::Write;
-
-use deno_core::{JsRuntime, RuntimeOptions};
-
-fn main() {
-    println!("cargo:rerun-if-changed=build.rs");
-    let options = RuntimeOptions {
-        will_snapshot: true,
-        ..Default::default()
-    };
-    let mut runtime = JsRuntime::new(options);
-
-    let mut snap = File::create("v8.snap").expect("can create snap file");
-    snap.write_all(&runtime.snapshot())
-        .expect("can write content to snap");
-}

backend/migrations/20220123221903_first.up.sql

@@ -2,6 +2,7 @@
 create SCHEMA IF NOT exists extensions;
 create extension if not exists "uuid-ossp"      with schema extensions;
 
+
 CREATE TABLE workspace (
     id VARCHAR(50) PRIMARY KEY,
     name VARCHAR(50) NOT NULL,
@@ -205,12 +206,6 @@ CREATE TABLE password (
     company VARCHAR(30)
 );
 
--- CREATE TABLE invite_code (
---     code VARCHAR(20) PRIMARY KEY,
---     seats_left INTEGER NOT NULL DEFAULT 0,
---     seats_given INTEGER NOT NULL DEFAULT 1
--- );
-
 
 CREATE TABLE workspace_settings (
     workspace_id VARCHAR(50) PRIMARY KEY REFERENCES workspace(id),
@@ -277,17 +272,6 @@ CREATE TABLE variable (
     CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
 );
 
--- CREATE TABLE oauth(
---     id VARCHAR(150) NOT NULL PRIMARY KEY,
---     owner VARCHAR(50),
---     workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
---     type VARCHAR(50) NOT NULL,
---     refresh_token VARCHAR(255), 
---     access_token VARCHAR(255) NOT NULL
--- );
-
--- CREATE INDEX index_oauth ON oauth (workspace_id, type, owner);
-
 CREATE TYPE ACTION_KIND AS ENUM ('create', 'update', 'delete', 'execute');
 
 CREATE TABLE audit (
@@ -420,35 +404,6 @@ CREATE INDEX worker_ping_on_ping_at ON worker_ping (ping_at);
 ALTER TABLE audit ENABLE ROW LEVEL SECURITY;
 CREATE POLICY audit_log_see_own ON audit FOR SELECT
 USING(audit.username = current_setting('session.user') or current_setting('session.is_admin')::boolean);
--- USING(current_setting('session.is_admin')::boolean);
-
-
-DO
-$do$
-BEGIN
-   IF NOT EXISTS (
-      SELECT FROM pg_catalog.pg_roles
-      WHERE  rolname = 'app') THEN
-
-      CREATE ROLE app LOGIN PASSWORD 'changeme';
-   END IF;
-END
-$do$;
-
-GRANT SELECT ON audit TO app;
-
-REVOKE ALL
-ON ALL TABLES IN SCHEMA public 
-FROM PUBLIC;
-
-GRANT ALL
-ON ALL TABLES IN SCHEMA public 
-TO admin;
-
-ALTER DEFAULT PRIVILEGES 
-    FOR ROLE admin
-    IN SCHEMA public
-    GRANT ALL ON TABLES TO admin;
 
 
 INSERT INTO usr_to_group
@@ -456,12 +411,10 @@ SELECT workspace_id, 'all', username FROM (SELECT workspace_id, username from us
 ;
 
 DROP POLICY audit_log_see_own on audit;
-GRANT ALL ON audit TO app;
 CREATE POLICY see_own ON audit FOR ALL
 USING (audit.username = current_setting('session.user'));
 
 
-GRANT ALL ON queue TO app;
 ALTER TABLE queue ENABLE ROW LEVEL SECURITY;
 
 CREATE POLICY see_own ON queue FOR ALL
@@ -470,7 +423,6 @@ USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(queue.perm
 CREATE POLICY see_member ON queue FOR ALL
 USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(queue.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
 
-GRANT ALL ON completed_job TO app;
 ALTER TABLE completed_job ENABLE ROW LEVEL SECURITY;
 
 
@@ -483,16 +435,6 @@ USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(co
 CREATE POLICY see_member ON completed_job FOR ALL
 USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(completed_job.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
 
-GRANT SELECT ON pipenv to app;
-GRANT SELECT (email, username, is_admin, workspace_id) ON usr to app;
-
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to app;
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to admin;
-GRANT SELECT, INSERT ON resource_type to app;
-
-GRANT SELECT ON worker_ping to app;
-GRANT SELECT ON worker_ping to admin;
-
 CREATE POLICY schedule ON audit FOR INSERT
 WITH CHECK (audit.username LIKE 'schedule-%');
 
@@ -508,7 +450,6 @@ $do$
     EXECUTE FORMAT(
       $$
 
-        GRANT ALL ON %1$I TO app;
         ALTER TABLE %1$I ENABLE ROW LEVEL SECURITY;
 
         CREATE POLICY see_starter ON %1$I FOR SELECT
@@ -542,13 +483,11 @@ $do$
   END
 $do$;
 
-GRANT ALL ON group_ TO app;
 ALTER TABLE group_
 ADD COLUMN extra_perms JSONB NOT NULL DEFAULT '{}';
 
 CREATE INDEX group_extra_perms ON group_ USING GIN (extra_perms);
 
-GRANT ALL ON usr_to_group TO app;
 ALTER TABLE usr_to_group ENABLE ROW LEVEL SECURITY;
 
 CREATE POLICY see_extra_perms_user ON usr_to_group FOR ALL
@@ -566,10 +505,62 @@ WITH CHECK (exists(
 DO
 $do$
 BEGIN
-   IF NOT EXISTS (
-      SELECT FROM pg_catalog.pg_roles  -- SELECT list can be empty for this
-      WHERE  rolname = 'admin') THEN
-      CREATE ROLE admin WITH BYPASSRLS LOGIN PASSWORD 'changeme';
-   END IF;
+    IF EXISTS (
+        select usesuper from pg_user where usename = CURRENT_USER AND usesuper = 't') 
+    AND NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'windmill_user') THEN
+
+        LOCK TABLE pg_catalog.pg_roles;
+
+        CREATE ROLE windmill_user;
+
+        GRANT ALL
+        ON ALL TABLES IN SCHEMA public 
+        TO windmill_user;
+
+        GRANT ALL PRIVILEGES 
+        ON ALL SEQUENCES IN SCHEMA public 
+        TO windmill_user;
+
+        ALTER DEFAULT PRIVILEGES 
+            IN SCHEMA public
+            GRANT ALL ON TABLES TO windmill_user;
+
+        ALTER DEFAULT PRIVILEGES 
+            IN SCHEMA public
+            GRANT ALL ON SEQUENCES TO windmill_user;
+
+    END IF;
+END
+$do$;
+
+DO
+$do$
+BEGIN
+    IF EXISTS (select usesuper from pg_user where usename = CURRENT_USER AND usesuper = 't')
+    AND NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'windmill_admin') THEN
+        CREATE ROLE windmill_admin WITH BYPASSRLS;
+
+        GRANT ALL
+        ON ALL TABLES IN SCHEMA public 
+        TO windmill_admin;
+
+        GRANT ALL PRIVILEGES 
+        ON ALL SEQUENCES IN SCHEMA public 
+        TO windmill_admin;
+
+        ALTER DEFAULT PRIVILEGES 
+            IN SCHEMA public
+            GRANT ALL ON TABLES TO windmill_admin;
+
+        ALTER DEFAULT PRIVILEGES 
+            IN SCHEMA public
+            GRANT ALL ON SEQUENCES TO windmill_admin;
+    END IF;
 END
 $do$;

backend/migrations/20220321004844_flow_preview.down.sql

@@ -1 +1,2 @@
 -- Add down migration script here
+

backend/migrations/20220428085013_private_key.up.sql

@@ -9,8 +9,5 @@ CREATE TABLE workspace_key (
     PRIMARY KEY (workspace_id, kind)
 );
 
-GRANT SELECT ON workspace_key TO app;
-GRANT SELECT ON workspace_key TO admin;
-
 INSERT INTO workspace_key SELECT id as workspace_id, 'cloud' as kind, 'changeme' as key FROM workspace;
 

backend/migrations/20220504193929_typescript_support.down.sql

@@ -0,0 +1,12 @@
+-- Add down migration script here
+DROP TYPE SCRIPT_LANG;
+
+ALTER TABLE script
+DROP COLUMN language SCRIPT_LANG;
+
+ALTER TABLE queue
+DROP COLUMN language SCRIPT_LANG;
+
+ALTER TABLE completed_job
+DROP COLUMN language SCRIPT_LANG;
+

backend/migrations/20220504193929_typescript_support.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+CREATE TYPE SCRIPT_LANG AS ENUM ('python3', 'deno');
+
+ALTER TABLE script
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
+
+ALTER TABLE queue
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
+
+ALTER TABLE completed_job
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';

backend/migrations/20220508150023_changeme_passwords.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508150023_changeme_passwords.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+UPDATE password 
+SET password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'
+WHERE password_hash = '$argon2id$v=19$m=4096,t=3,p=1$z0Kg3qyaS14e+YHeihkJLQ$N69flI6yQ/U98pjAHtbNxbdz2f4PrJEi9Tx1VoYk1as';

backend/migrations/20220508152326_create_role_if_not_exists.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508152326_create_role_if_not_exists.up.sql

@@ -0,0 +1 @@
+-- Add up migration script here

backend/migrations/20220508155403_delete_starter_scripts.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508155403_delete_starter_scripts.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+DELETE FROM script WHERE lock IS NULL;
+

backend/migrations/20220509191845_started_at_completed_job.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE completed_job
+DROP COLUMN started_at;

backend/migrations/20220509191845_started_at_completed_job.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE completed_job
+ADD COLUMN started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW();
+

backend/migrations/20220518211432_make_lang_nullable.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220518211432_make_lang_nullable.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+ALTER TABLE queue
+ALTER COLUMN language DROP NOT NULL;
+
+ALTER TABLE completed_job
+ALTER COLUMN language DROP NOT NULL;

backend/migrations/20220610181005_add_script_hub.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220610181005_add_script_hub.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE JOB_KIND ADD VALUE 'script_hub';

backend/migrations/20220615151034_accounts.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220615151034_accounts.up.sql

@@ -0,0 +1,13 @@
+-- Add up migration script here
+
+CREATE TABLE account (
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    id SERIAL NOT NULL,
+    expires_at TIMESTAMP,
+    refresh_token VARCHAR(255),
+    PRIMARY KEY (workspace_id, id)
+);
+
+ALTER TABLE resource ADD COLUMN account INTEGER;
+ALTER TABLE variable ADD COLUMN account INTEGER;
+ALTER TABLE password ALTER COLUMN login_type TYPE VARCHAR(50);

backend/migrations/20220620210708_regex_fix.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220620210708_regex_fix.up.sql

@@ -0,0 +1,9 @@
+-- Add up migration script here
+
+ALTER TABLE usr DROP CONSTRAINT proper_email;
+ALTER TABLE usr ADD CONSTRAINT proper_email
+CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');
+
+ALTER TABLE workspace_invite DROP CONSTRAINT proper_email;
+ALTER TABLE workspace_invite ADD CONSTRAINT proper_email
+CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');

backend/migrations/20220624170622_workspace_premium.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220624170622_workspace_premium.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace ADD COLUMN premium BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20220624180013_workspace_premium_grant.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220624180013_workspace_premium_grant.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+

backend/migrations/20220707094935_add_client_to_account.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220707094935_add_client_to_account.up.sql

@@ -0,0 +1,20 @@
+-- Add up migration script here
+ALTER TABLE account ADD COLUMN owner VARCHAR(50) NOT NULL;
+ALTER TABLE account ADD COLUMN client VARCHAR(50) NOT NULL;
+ALTER TABLE resource ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE variable ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE resource DROP COLUMN account;
+
+ALTER TABLE account ALTER COLUMN expires_at TYPE TIMESTAMP WITH TIME ZONE;
+
+ALTER TABLE account ALTER COLUMN expires_at SET NOT NULL;
+ALTER TABLE account ALTER COLUMN refresh_token SET NOT NULL;
+
+ALTER TABLE account ENABLE ROW LEVEL SECURITY;
+
+
+CREATE POLICY see_own ON account FOR ALL
+USING (SPLIT_PART(account.owner, '/', 1) = 'u' AND SPLIT_PART(account.owner, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON account FOR ALL
+USING (SPLIT_PART(account.owner, '/', 1) = 'g' AND SPLIT_PART(account.owner, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20220713142758_script_trigger.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220713142758_script_trigger.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN trigger_reco_interval INTEGER;
+ALTER TABLE completed_job ADD COLUMN is_skipped BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20220719101639_script_trigger_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220719101639_script_trigger_2.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE script DROP COLUMN trigger_reco_interval;
+ALTER TABLE script ADD COLUMN is_trigger BOOLEAN NOT NULL DEFAULT false;
+

backend/migrations/20220728222351_job_duration_ms.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220728222351_job_duration_ms.up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE completed_job
+     RENAME duration to duration_ms;
+UPDATE completed_job
+   SET duration_ms = duration_ms * 1000;

backend/migrations/20220816185849_remove_non_admin_users.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220816185849_remove_non_admin_users.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+DELETE FROM password WHERE email = 'user@windmill.dev' OR email = 'ruben@windmill.dev';

backend/migrations/20220818125523_longername.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220818125523_longername.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE queue ALTER COLUMN created_by TYPE varchar(255);
+ALTER TABLE completed_job ALTER COLUMN created_by TYPE varchar(255);

backend/oauth_connect.json

@@ -0,0 +1,94 @@
+{
+    "github": {
+        "auth_url": "https://github.com/login/oauth/authorize",
+        "token_url": "https://github.com/login/oauth/access_token",
+        "scopes": [
+            "workflow",
+            "repo"
+        ]
+    },
+    "gitlab": {
+        "auth_url": "https://gitlab.com/oauth/authorize",
+        "token_url": "https://gitlab.com/oauth/token",
+        "scopes": [
+            "api"
+        ]
+    },
+    "bitbucket": {
+        "auth_url": "https://bitbucket.org/site/oauth2/authorize",
+        "token_url": "https://bitbucket.org/site/oauth2/access_token",
+        "scopes": [
+            "repository"
+        ]
+    },
+    "slack": {
+        "auth_url": "https://slack.com/oauth/authorize",
+        "token_url": "https://slack.com/api/oauth.access",
+        "scopes": [
+            "chat:write:user"
+        ]
+    },
+    "gsheets": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/spreadsheets"
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "gdrive": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/drive"
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "gmail": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/gmail.send"
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "gcal": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/calendar.events"
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "gcloud": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform"
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "basecamp": {
+        "auth_url": "https://launchpad.37signals.com/authorization/new",
+        "token_url": "https://launchpad.37signals.com/authorization/token",
+        "scopes": [],
+        "extra_params": {
+            "type": "web_server"
+        }
+    }
+}

backend/oauth_login.json

@@ -0,0 +1,23 @@
+{
+    "github": {
+        "auth_url": "https://github.com/login/oauth/authorize",
+        "token_url": "https://github.com/login/oauth/access_token",
+        "scopes": [
+            "user:email"
+        ]
+    },
+    "gitlab": {
+        "auth_url": "https://gitlab.com/oauth/authorize",
+        "token_url": "https://gitlab.com/oauth/token",
+        "scopes": [
+            "read_user"
+        ]
+    },
+    "google": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/userinfo.email"
+        ]
+    }
+}

backend/overwrite_migrations.py

@@ -0,0 +1,24 @@
+import sys
+import os
+import subprocess
+
+database_url = sys.argv[1]
+
+print(f"database_url: {database_url}")
+
+if database_url is None:
+    print("Please provide a database url")
+    sys.exit(1)
+
+for f in os.listdir("migrations"):
+    if f.endswith(".up.sql"):
+        version = f.split("_")[0]
+        cmd = f"cat migrations/{f} | openssl dgst -sha384 | cut -d ' ' -f 2"
+        ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+        digest = ps.communicate()[0].decode("utf-8").strip()
+
+        cmd = f"psql '{database_url}' -c \"UPDATE _sqlx_migrations SET checksum = '\\x{digest}' WHERE version = {version};\""
+        ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+        out = ps.communicate()[0].decode("utf-8").strip()
+        print(version, digest, out)
+

backend/rustfmt.toml

@@ -1,7 +1,5 @@
-imports_granularity = "Crate"
 max_width = 100
 use_small_heuristics = "Default"
-indent_style = "Block"
-fn_single_line = false
-force_multiline_blocks = true
-format_strings = true
+match_arm_leading_pipes="Preserve"
+struct_lit_width=100
+struct_variant_width=100

backend/src/audit.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -64,13 +65,12 @@ pub async fn audit_log<'c>(
     let p_json: serde_json::Value = serde_json::to_value(&parameters).unwrap();
 
     tracing::info!(
-        username = username,
-        kind = "audit",
         operation = operation,
-        workspace = w_id,
         action_kind = ?action_kind,
         resource = resource,
-        parameters = %p_json
+        parameters = %p_json,
+        workspace_id = w_id,
+        username = username,
     );
     sqlx::query(
         "INSERT INTO audit

backend/src/client.rs

@@ -31,14 +31,17 @@ pub async fn get_resource(
     base_url: &str,
 ) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let client = reqwest::Client::new();
-    let result = client
+    let res = client
         .get(format!(
             "{base_url}/api/w/{workspace}/resources/get_value/{path}"
         ))
         .bearer_auth(token)
         .send()
-        .await?
-        .json::<Option<serde_json::Value>>()
         .await?;
-    Ok(result)
+    if res.status().is_success() {
+        let value = res.json::<Option<serde_json::Value>>().await?;
+        Ok(value)
+    } else {
+        Err(Error::NotFound(format!("Resource not found at {path}")))?
+    }
 }

backend/src/db.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -11,9 +12,9 @@ use std::time::Duration;
 
 pub type DB = Pool<Postgres>;
 
-pub async fn connect(database_url: &str) -> Result<DB, Error> {
+pub async fn connect(database_url: &str, max_connections: u32) -> Result<DB, Error> {
     PgPoolOptions::new()
-        .max_connections(100)
+        .max_connections(max_connections)
         .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
         .connect(database_url)
         .await
@@ -29,19 +30,6 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn setup_app_user(db: &DB, password: &str) -> Result<(), Error> {
-    let mut tx = db.begin().await?;
-
-    sqlx::query(&format!("ALTER USER app WITH PASSWORD '{}'", password))
-        .execute(&mut tx)
-        .await?;
-    sqlx::query(&format!("ALTER USER admin WITH PASSWORD '{}'", password))
-        .execute(&mut tx)
-        .await?;
-    tx.commit().await?;
-
-    Ok(())
-}
 #[derive(Clone)]
 pub struct UserDB {
     db: DB,
@@ -57,9 +45,13 @@ impl UserDB {
         authed: &Authed,
     ) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
         let mut tx = self.db.begin().await?;
-        let user = if authed.is_admin { "admin" } else { "app" };
+        let user = if authed.is_admin {
+            "windmill_admin"
+        } else {
+            "windmill_user"
+        };
 
-        sqlx::query(&format!("SET LOCAL SESSION AUTHORIZATION {}", user))
+        sqlx::query(&format!("SET LOCAL ROLE {}", user))
             .execute(&mut tx)
             .await?;
 

backend/src/error.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -44,7 +45,7 @@ pub enum Error {
     HexErr(#[from] hex::FromHexError),
     #[error("Migrating database: {0}")]
     DatabaseMigration(#[from] MigrateError),
-    #[error("{0}")]
+    #[error(transparent)]
     Anyhow(#[from] anyhow::Error),
 }
 
@@ -62,6 +63,11 @@ impl IntoResponse for Error {
             Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST,
             _ => StatusCode::INTERNAL_SERVER_ERROR,
         };
-        Response::builder().status(status).body(body).unwrap()
+        tracing::error!(error = e.to_string());
+        Response::builder()
+            .header("Content-Type", "text/plain")
+            .status(status)
+            .body(body)
+            .unwrap()
     }
 }

backend/src/external_ip.rs

@@ -0,0 +1,14 @@
+//! Used to determine the internet address that connections from workers will appear to come from.
+//!
+//! For users writing scripts to access their infrastructure with firewalls requiring incoming
+//! connections to be from whitelisted IP addresses.
+
+use reqwest::Result;
+
+pub async fn get_ip() -> Result<String> {
+    reqwest::get("https://hub.windmill.dev/getip")
+        .await?
+        .error_for_status()?
+        .text()
+        .await
+}

backend/src/fixtures/base.sql

@@ -0,0 +1,19 @@
+-- used for backend automated testing
+-- https://docs.rs/sqlx/latest/sqlx/attr.test.html
+
+INSERT INTO workspace
+            (id,               name,             owner,       domain)
+     VALUES ('test-workspace', 'test-workspace', 'test-user', null);
+
+CREATE FUNCTION "notify_insert_on_completed_job" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('insert on completed_job', NEW.id::text);
+    RETURN new;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_insert_on_completed_job"
+ AFTER INSERT ON "completed_job"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_insert_on_completed_job" ();

backend/src/flows.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -7,24 +8,26 @@
 
 use std::collections::HashMap;
 
+use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
-    extract::{Extension, Path, Query},
+    extract::{Extension, Host, Path, Query},
     routing::{get, post},
     Json, Router,
 };
 use serde::{Deserialize, Serialize};
 use sql_builder::SqlBuilder;
-use sqlx::FromRow;
+use sqlx::{FromRow, Postgres, Transaction};
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::UserDB,
-    error::{Error, JsonResult, Result},
+    db::{UserDB, DB},
+    error::{self, to_anyhow, Error, JsonResult, Result},
+    jobs::RawCode,
     scripts::Schema,
     users::Authed,
-    utils::{Pagination, StripPath},
+    utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath},
 };
 
 pub fn workspaced_service() -> Router {
@@ -34,6 +37,13 @@ pub fn workspaced_service() -> Router {
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
+        .route("/exists/*path", get(exists_flow_by_path))
+}
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/hub/list", get(list_hub_flows))
+        .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
 #[derive(FromRow, Serialize)]
@@ -59,19 +69,24 @@ pub struct NewFlow {
     pub schema: Option<Schema>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct FlowValue {
     pub modules: Vec<FlowModule>,
     pub failure_module: Option<FlowModule>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct FlowModule {
-    pub input_transform: HashMap<String, InputTransform>,
+    #[serde(default)]
+    #[serde(alias = "input_transform")]
+    pub input_transforms: HashMap<String, InputTransform>,
     pub value: FlowModuleValue,
+    pub stop_after_if_expr: Option<String>,
+    pub skip_if_stopped: Option<bool>,
+    pub summary: Option<String>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
@@ -79,17 +94,31 @@ pub struct FlowModule {
 pub enum InputTransform {
     Static { value: serde_json::Value },
     Javascript { expr: String },
-    Resource { path: String },
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
 )]
 pub enum FlowModuleValue {
-    Script { path: String },
-    Flow { path: String },
+    Script {
+        path: String,
+    },
+    ForloopFlow {
+        iterator: InputTransform,
+        value: Box<FlowValue>,
+        #[serde(default = "default_true")]
+        skip_failures: bool,
+    },
+    Flow {
+        path: String,
+    },
+    RawScript(RawCode),
+}
+
+fn default_true() -> bool {
+    true
 }
 
 #[derive(Deserialize)]
@@ -121,7 +150,7 @@ async fn list_flows(
             "edited_by",
             "edited_at",
             "archived",
-            "schema",
+            "null schema",
             "extra_perms",
         ])
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
@@ -150,6 +179,43 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
+async fn list_hub_flows(
+    Authed { email, username, .. }: Authed,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let flows = list_elems_from_hub(
+        http_client,
+        "https://hub.windmill.dev/searchFlowData?approved=true",
+        email,
+        username,
+        host,
+    )
+    .await?;
+    Ok(Json(flows))
+}
+
+pub async fn get_hub_flow_by_id(
+    Authed { email, username, .. }: Authed,
+    Path(id): Path<i32>,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let value = http_get_from_hub(
+        http_client,
+        &format!("https://hub.windmill.dev/flows/{id}/json"),
+        email,
+        username,
+        host,
+        false,
+    )
+    .await?
+    .json()
+    .await
+    .map_err(to_anyhow)?;
+    Ok(Json(value))
+}
+
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -159,15 +225,17 @@ async fn create_flow(
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
+    check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+
     sqlx::query!(
-        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::json)",
+        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
+         schema) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json)",
         w_id,
         nf.path,
         nf.summary,
         nf.description,
         nf.value,
         &authed.username,
-        &chrono::Utc::now(),
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
     )
     .execute(&mut tx)
@@ -193,6 +261,29 @@ async fn create_flow(
     Ok(nf.path.to_string())
 }
 
+async fn check_schedule_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> error::Result<()> {
+    let exists_flow = sqlx::query_scalar!(
+        "SELECT EXISTS (SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2 AND path != \
+         script_path)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists_flow {
+        return Err(error::Error::BadConfig(format!(
+            "A flow cannot have the same path as a schedule if the schedule does not trigger that \
+             same flow: {path}",
+        )));
+    };
+    Ok(())
+}
+
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -202,15 +293,17 @@ async fn update_flow(
     let mut tx = user_db.begin(&authed).await?;
 
     let flow_path = flow_path.to_path();
+    check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
+
     let schema = nf.schema.map(|x| x.0);
     let flow = sqlx::query_scalar!(
-        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = $6, schema = $7 WHERE path = $8 AND workspace_id = $9 RETURNING path",
+        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
+         edited_at = now(), schema = $6 WHERE path = $7 AND workspace_id = $8 RETURNING path",
         nf.path,
         nf.summary,
         nf.description,
         nf.value,
         &authed.username,
-        &chrono::Utc::now(),
         schema,
         flow_path,
         w_id,
@@ -260,6 +353,25 @@ async fn get_flow_by_path(
     Ok(Json(flow))
 }
 
+async fn exists_flow_by_path(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id \
+         = 'starter'))",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -302,25 +414,78 @@ mod tests {
         let mut hm = HashMap::new();
         hm.insert(
             "test".to_owned(),
-            InputTransform::Static {
-                value: serde_json::json!("test2"),
-            },
+            InputTransform::Static { value: serde_json::json!("test2") },
         );
         let fv = FlowValue {
-            modules: vec![FlowModule {
-                input_transform: hm,
-                value: FlowModuleValue::Script {
-                    path: "test".to_string(),
+            modules: vec![
+                FlowModule {
+                    input_transforms: hm,
+                    value: FlowModuleValue::Script { path: "test".to_string() },
+                    stop_after_if_expr: None,
+                    skip_if_stopped: Some(false),
+                    summary: None,
                 },
-            }],
+                FlowModule {
+                    input_transforms: HashMap::new(),
+                    value: FlowModuleValue::RawScript(RawCode {
+                        content: "test".to_string(),
+                        language: crate::scripts::ScriptLang::Deno,
+                        path: None,
+                    }),
+                    stop_after_if_expr: Some("foo = 'bar'".to_string()),
+                    skip_if_stopped: None,
+                    summary: None,
+                },
+                FlowModule {
+                    input_transforms: [(
+                        "iterand".to_string(),
+                        InputTransform::Static { value: serde_json::json!(vec![1, 2, 3]) },
+                    )]
+                    .into(),
+                    value: FlowModuleValue::ForloopFlow {
+                        iterator: InputTransform::Static { value: serde_json::json!([1, 2, 3]) },
+                        value: Box::new(FlowValue { modules: vec![], failure_module: None }),
+                        skip_failures: true,
+                    },
+                    stop_after_if_expr: Some("previous.isEmpty()".to_string()),
+                    skip_if_stopped: None,
+                    summary: None,
+                },
+            ],
             failure_module: Some(FlowModule {
-                input_transform: HashMap::new(),
-                value: FlowModuleValue::Flow {
-                    path: "test".to_string(),
-                },
+                input_transforms: HashMap::new(),
+                value: FlowModuleValue::Flow { path: "test".to_string() },
+                stop_after_if_expr: Some("previous.isEmpty()".to_string()),
+                skip_if_stopped: None,
+                summary: None,
             }),
         };
         println!("{}", serde_json::json!(fv).to_string());
         Ok(())
     }
+
+    #[test]
+    fn test_back_compat() {
+        /* renamed input_transform -> input_transforms but should deserialize old name */
+        let s = r#"
+        {
+            "value": {
+                "type": "rawscript",
+                "content": "def main(n): return",
+                "language": "python3"
+            },
+            "input_transform": {
+                "n": {
+                    "expr": "flow_input.iter.value",
+                    "type": "javascript"
+                }
+            }
+        }
+        "#;
+        let module: FlowModule = serde_json::from_str(s).unwrap();
+        assert_eq!(
+            module.input_transforms["n"],
+            InputTransform::Javascript { expr: "flow_input.iter.value".to_string() }
+        );
+    }
 }

backend/src/granular_acls.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -46,7 +47,8 @@ async fn add_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
+         true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(write.unwrap_or(false))
     .bind(path)
@@ -74,7 +76,8 @@ async fn remove_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
+         workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(owner)
     .bind(path)

backend/src/groups.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -258,9 +259,7 @@ async fn add_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username {
-        username: user_username,
-    }): Json<Username>,
+    Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
@@ -294,9 +293,7 @@ async fn remove_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username {
-        username: user_username,
-    }): Json<Username>,
+    Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 

backend/src/js_eval.rs

@@ -1,57 +1,60 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::cell::RefCell;
-use std::rc::Rc;
-
-use deno_core::serde_v8;
-use deno_core::v8;
-use deno_core::v8::IsolateHandle;
-use deno_core::JsRuntime;
-use deno_core::OpState;
-use deno_core::RuntimeOptions;
-use deno_core::Snapshot;
-use deno_core::ZeroCopyBuf;
+use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
 use itertools::Itertools;
 use regex::Regex;
 use serde_json::Value;
-use tokio::sync::oneshot;
-use tokio::time::timeout;
+use tokio::{sync::oneshot, time::timeout};
 
-use crate::client;
-use crate::error::Error;
+use crate::{client, error::Error};
+
+pub struct EvalCreds {
+    pub workspace: String,
+    pub token: String,
+}
 
 pub async fn eval_timeout(
     expr: String,
     env: Vec<(String, serde_json::Value)>,
-    workspace: &str,
-    token: &str,
+    creds: Option<EvalCreds>,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr2 = expr.clone();
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
-    let (workspace, token) = (workspace.to_string().clone(), token.to_string().clone());
     timeout(
         std::time::Duration::from_millis(2000),
         tokio::task::spawn_blocking(move || {
-            let buffer = include_bytes!("../v8.snap");
+            let mut ops = vec![];
 
+            if creds.is_some() {
+                ops.extend([
+                    // An op for summing an array of numbers
+                    // The op-layer automatically deserializes inputs
+                    // and serializes the returned Result & value
+                    op_variable::decl(),
+                    op_resource::decl(),
+                ])
+            }
+
+            if !steps.is_empty() {
+                ops.push(op_get_result::decl())
+            }
+
+            let ext = Extension::builder().ops(ops).build();
             // Use our snapshot to provision our new runtime
             let options = RuntimeOptions {
-                startup_snapshot: Some(Snapshot::Static(buffer)),
+                extensions: vec![ext],
+                //                startup_snapshot: Some(Snapshot::Static(buffer)),
                 ..Default::default()
             };
+
             let mut js_runtime = JsRuntime::new(options);
-            js_runtime.register_op("variable", deno_core::op_async(op_variable));
-            js_runtime.register_op("resource", deno_core::op_async(op_resource));
-            if !steps.is_empty() {
-                js_runtime.register_op("result", deno_core::op_async(op_get_result));
-            }
-            js_runtime.sync_ops_cache();
 
             sender
                 .send(js_runtime.v8_isolate().thread_safe_handle())
@@ -68,8 +71,7 @@ pub async fn eval_timeout(
                 .into_iter()
                 .fold(expr, replace_with_await);
 
-            let r =
-                runtime.block_on(eval(&mut js_runtime, &expr, env, &workspace, &token, steps))?;
+            let r = runtime.block_on(eval(&mut js_runtime, &expr, env, creds, steps))?;
 
             Ok(r) as anyhow::Result<Value>
         }),
@@ -114,13 +116,31 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
+pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
+    let mut context = JsRuntime::new(RuntimeOptions::default());
+    let code = format!("let x = {}; x", code);
+    let res = context.execute_script("<anon>", &code);
+    match res {
+        Ok(global) => {
+            let scope = &mut context.handle_scope();
+            let local = v8::Local::new(scope, global);
+            let deserialized_value = serde_v8::from_v8::<serde_json::Value>(scope, local);
+
+            match deserialized_value {
+                Ok(value) => Ok(value),
+                Err(err) => Err(format!("Cannot deserialize value: {:?}", err)),
+            }
+        }
+        Err(err) => Err(format!("Evaling error: {:?}", err)),
+    }
+}
+
 const SPLIT_PAT: &str = ";\n";
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
     env: Vec<(String, serde_json::Value)>,
-    workspace: &str,
-    token: &str,
+    creds: Option<EvalCreds>,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr = expr.trim();
@@ -131,14 +151,12 @@ async fn eval(
             .join("\n"),
         expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
     );
-    let steps_code = if !steps.is_empty() {
-        format!(
-            r#"
+    let (steps_code, api_code) = if let Some(EvalCreds { workspace, token }) = creds {
+        let steps_code = if !steps.is_empty() {
+            format!(
+                r#"
 let steps = [{}];
 async function step(n) {{
-    if (n == 0) {{
-        return flow_input;
-    }}
     if (n == -1) {{
         return previous_result;
     }}
@@ -148,42 +166,55 @@ async function step(n) {{
         n = n % steps.length + steps.length;
     }}
     let id = steps[n];
-    return await Deno.core.opAsync("result", [workspace, id, token, base_url]);
+    return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
 }}"#,
-            steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
-        )
-    } else {
-        "".to_string()
-    };
+                steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
+            )
+        } else {
+            String::new()
+        };
 
-    let code = format!(
-        r#"
+        let api_code = format!(
+            r#"
 let workspace = "{workspace}";
 let base_url = "{}";
 async function variable(path) {{
     let token = "{token}";
-    return await Deno.core.opAsync("variable", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_variable", [workspace, path, token, base_url]);
 }}
 async function resource(path) {{
     let token = "{token}";
-    return await Deno.core.opAsync("resource", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_resource", [workspace, path, token, base_url]);
 }}
+        "#,
+            std::env::var("BASE_INTERNAL_URL")
+                .unwrap_or_else(|_| "http://missing-base-url".to_string()),
+        );
+        (steps_code, api_code)
+    } else {
+        (String::new(), String::new())
+    };
+
+    let code = format!(
+        r#"
+{api_code}
 {}
 {steps_code}
 (async () => {{ 
     {expr} 
 }})()
         "#,
-        std::env::var("BASE_INTERNAL_URL")
-            .unwrap_or_else(|_| "http://missing-base-url".to_string()),
         env.into_iter()
-            .map(|(a, b)| format!(
-                "let {a} = {};\n",
-                serde_json::to_string(&b)
-                    .unwrap_or_else(|_| "\"error serializing value\"".to_string())
-            ))
+            .map(|(a, b)| {
+                format!(
+                    "let {a} = {};\n",
+                    serde_json::to_string(&b)
+                        .unwrap_or_else(|_| "\"error serializing value\"".to_string())
+                )
+            })
             .join(""),
     );
+    tracing::debug!("{}", code);
     let global = context.execute_script("<anon>", &code)?;
     let global = context.resolve_value(global).await?;
 
@@ -204,11 +235,8 @@ async function resource(path) {{
 //     Ok(path)
 // }
 
-async fn op_variable(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<String, anyhow::Error> {
+#[op]
+async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -216,11 +244,8 @@ async fn op_variable(
     client::get_variable(workspace, path, token, &base_url).await
 }
 
-async fn op_get_result(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<Option<serde_json::Value>, anyhow::Error> {
+#[op]
+async fn op_get_result(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let id = &args[1];
     let token = &args[2];
@@ -238,11 +263,8 @@ async fn op_get_result(
     Ok(result)
 }
 
-async fn op_resource(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<Option<serde_json::Value>, anyhow::Error> {
+#[op]
+async fn op_resource(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -267,7 +289,7 @@ mod tests {
         let code = "value.test + params.test";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
+        let res = eval(&mut runtime, code, env, None, vec![]).await?;
         assert_eq!(res, json!(4));
         Ok(())
     }
@@ -280,7 +302,7 @@ mod tests {
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
+        let res = eval(&mut runtime, code, env, None, vec![]).await?;
         assert_eq!(res, json!("my 5\nmultiline template"));
         Ok(())
     }
@@ -291,10 +313,10 @@ multiline template`";
             ("params".to_string(), json!({"test": 2})),
             ("value".to_string(), json!({"test": 2})),
         ];
-        let code = r#"variable("test")"#;
+        let code = r#"params.test"#;
 
-        let res = eval_timeout(code.to_string(), env, "workspace", "token", vec![]).await?;
-        assert_eq!(res, json!("test"));
+        let res = eval_timeout(code.to_string(), env, None, vec![]).await?;
+        assert_eq!(res, json!(2));
         Ok(())
     }
 }

backend/src/lib.rs

@@ -1,24 +1,24 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use ::oauth2::basic::BasicClient;
+use anyhow::Context;
 use argon2::Argon2;
-use axum::{extract::extractor_middleware, handler::Handler, routing::get, Extension, Router};
+use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
+use futures::FutureExt;
 use git_version::git_version;
-use hyper::Response;
+use rand::Rng;
 use slack_http_verifier::SlackVerifier;
-use std::{collections::HashMap, net::SocketAddr, sync::Arc};
-use tokio::sync::Mutex;
+use std::{net::SocketAddr, sync::Arc};
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
-use tower_http::trace::{MakeSpan, OnResponse, TraceLayer};
-use tracing::{field, Span};
-use tracing_subscriber::{filter::filter_fn, prelude::*, EnvFilter};
+use tower_http::trace::TraceLayer;
+
 extern crate magic_crypt;
 
 extern crate dotenv;
@@ -28,126 +28,68 @@ mod client;
 mod db;
 mod email;
 mod error;
-mod flow;
+mod external_ip;
+mod flows;
 mod granular_acls;
 mod groups;
 mod jobs;
 mod js_eval;
 mod oauth2;
 mod parser;
+mod parser_py;
+mod parser_ts;
 mod resources;
 mod schedule;
 mod scripts;
 mod static_assets;
+mod tracing_init;
 mod users;
 mod utils;
 mod variables;
 mod worker;
+mod worker_flow;
 mod worker_ping;
 mod workspaces;
 
 use error::Error;
 
 pub use crate::email::EmailSender;
-use crate::{db::UserDB, utils::rd_string};
+use crate::{
+    db::UserDB,
+    error::to_anyhow,
+    oauth2::build_oauth_clients,
+    tracing_init::{MyMakeSpan, MyOnResponse},
+    utils::rd_string,
+};
+
+pub use crate::tracing_init::initialize_tracing;
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 pub const DEFAULT_NUM_WORKERS: usize = 3;
 pub const DEFAULT_TIMEOUT: i32 = 300;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
-
-#[derive(Clone)]
-struct MyOnResponse {}
-
-impl<B> OnResponse<B> for MyOnResponse {
-    fn on_response(
-        self,
-        response: &Response<B>,
-        latency: std::time::Duration,
-        _span: &tracing::Span,
-    ) {
-        tracing::info!(
-            latency = %latency.as_millis(),
-            status = ?response.status(),
-            "finished processed request")
-    }
-}
-
-#[derive(Clone)]
-struct MyMakeSpan {}
-
-impl<B> MakeSpan<B> for MyMakeSpan {
-    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
-        tracing::info_span!(
-            "request",
-            method = %request.method(),
-            uri = %request.uri(),
-            version = ?request.version(),
-            username = field::Empty,
-        )
-    }
-}
-
-pub async fn initialize_tracing() -> anyhow::Result<()> {
-    //let log_level = if std::env::var("RUST_LOG").map(|x| &x == "debug")
-    let ts_base = tracing_subscriber::registry()
-        .with(
-            EnvFilter::from_default_env()
-                //.add_directive("windmill".parse()?)
-                .add_directive("runtime=trace".parse()?)
-                .add_directive("tokio=trace".parse()?),
-        )
-        .with(
-            tracing_subscriber::fmt::layer()
-                .json()
-                .flatten_event(true)
-                .with_span_list(false)
-                .with_current_span(true)
-                .with_filter(filter_fn(|meta| meta.target().starts_with("windmill"))),
-        );
-
-    if std::env::var("TOKIO_CONSOLE")
-        .map(|x| x == "true")
-        .unwrap_or(false)
-    {
-        let console_layer = console_subscriber::spawn();
-        ts_base.with(console_layer).init();
-    } else {
-        ts_base.init();
-    }
-    Ok(())
-}
+pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
 
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
-    let app_password = std::env::var("APP_USER_PASSWORD").unwrap_or_else(|_| "changeme".to_owned());
-
     db::migrate(db).await?;
-    db::setup_app_user(db, &app_password).await?;
     Ok(())
 }
 
 pub async fn connect_db() -> anyhow::Result<DB> {
     let database_url = std::env::var("DATABASE_URL")
         .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-    Ok(db::connect(&database_url).await?)
+
+    let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
+        Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
+        Err(_) => DEFAULT_MAX_CONNECTIONS,
+    };
+
+    Ok(db::connect(&database_url, max_connections).await?)
 }
 
-type BasicClientsMap = HashMap<String, BasicClient>;
-
-pub fn build_oauth_clients(base_url: &str) -> BasicClientsMap {
-    [(
-        "github".to_string(),
-        oauth2::build_gh_client(
-            &std::env::var("GITHUB_OAUTH_CLIENT_ID").unwrap_or_else(|_| "".to_string()),
-            &std::env::var("GITHUB_OAUTH_CLIENT_SECRET").unwrap_or_else(|_| "".to_string()),
-            base_url,
-        ),
-    )]
-    .into()
-}
-
-#[derive(Clone)]
 struct BaseUrl(String);
+struct IsSecure(bool);
+struct CloudHosted(bool);
 
 pub async fn run_server(
     db: DB,
@@ -161,13 +103,16 @@ pub async fn run_server(
     let auth_cache = Arc::new(users::AuthCache::new(db.clone()));
     let argon2 = Arc::new(Argon2::default());
     let email_sender = Arc::new(es);
-    let basic_clients = Arc::new(build_oauth_clients(base_url));
+    let basic_clients = Arc::new(build_oauth_clients(base_url).await?);
     let slack_verifier = Arc::new(
         std::env::var("SLACK_SIGNING_SECRET")
             .ok()
             .map(|x| SlackVerifier::new(x).unwrap()),
     );
-
+    let http_client = reqwest::ClientBuilder::new()
+        .user_agent("windmill/beta")
+        .build()
+        .map_err(to_anyhow)?;
     let middleware_stack = ServiceBuilder::new()
         .layer(
             TraceLayer::new_for_http()
@@ -179,7 +124,14 @@ pub async fn run_server(
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(Extension(basic_clients))
-        .layer(Extension(BaseUrl(base_url.to_string())))
+        .layer(Extension(Arc::new(BaseUrl(base_url.to_string()))))
+        .layer(Extension(Arc::new(CloudHosted(
+            std::env::var("CLOUD_HOSTED").is_ok(),
+        ))))
+        .layer(Extension(Arc::new(IsSecure(
+            base_url.starts_with("https://"),
+        ))))
+        .layer(Extension(http_client))
         .layer(CookieManagerLayer::new());
     // build our application with a route
     let app = Router::new()
@@ -205,7 +157,7 @@ pub async fn run_server(
                         .nest("/audit", audit::workspaced_service())
                         .nest("/acls", granular_acls::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/flows", flow::workspaced_service()),
+                        .nest("/flows", flows::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
@@ -214,9 +166,10 @@ pub async fn run_server(
                 )
                 .nest("/workers", worker_ping::global_service())
                 .nest("/scripts", scripts::global_service())
+                .nest("/flows", flows::global_service())
                 .nest("/schedules", schedule::global_service())
-                .route_layer(extractor_middleware::<users::Authed>())
-                .route_layer(extractor_middleware::<users::Tokened>())
+                .route_layer(from_extractor::<users::Authed>())
+                .route_layer(from_extractor::<users::Tokened>())
                 .nest(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),
@@ -247,14 +200,13 @@ pub async fn run_server(
     Ok(())
 }
 
-pub fn monitor_db(db: &DB, timeout: i32, tx: tokio::sync::broadcast::Sender<()>) {
+pub fn monitor_db(db: &DB, timeout: i32, rx: tokio::sync::broadcast::Receiver<()>) {
     let db1 = db.clone();
     let db2 = db.clone();
 
-    let rx1 = tx.subscribe();
-    let rx2 = tx.subscribe();
+    let rx2 = rx.resubscribe();
 
-    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx1).await });
+    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx).await });
     tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
@@ -265,32 +217,27 @@ pub async fn run_workers(
     num_workers: i32,
     sleep_queue: u64,
     base_url: String,
-    tx: tokio::sync::broadcast::Sender<()>,
+    disable_nuser: bool,
+    disable_nsjail: bool,
+    rx: tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
     let instance_name = rd_string(5);
 
-    let mutex = Arc::new(Mutex::new(0));
-
-    let sources: external_ip::Sources = external_ip::get_http_sources();
-    let consensus = external_ip::ConsensusBuilder::new()
-        .add_sources(sources)
-        .build();
-
-    let ip = consensus
-        .get_consensus()
-        .await
-        .map(|x| x.to_string())
-        .unwrap_or_else(|| "Unretrievable ip".to_string());
+    let ip = external_ip::get_ip().await.unwrap_or_else(|e| {
+        tracing::warn!(error = e.to_string(), "failed to get external IP");
+        "unretrievable IP".to_string()
+    });
 
     let mut handles = Vec::new();
+    let mut rng = rand::thread_rng();
+
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
-        let m1 = mutex.clone();
         let ip = ip.clone();
-        let tx = tx.clone();
         let base_url = base_url.clone();
+        let rx = rx.resubscribe();
         handles.push(tokio::spawn(async move {
             tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
             worker::run_worker(
@@ -300,14 +247,20 @@ pub async fn run_workers(
                 worker_name,
                 i as u64,
                 num_workers as u64,
-                m1,
                 &ip,
                 sleep_queue,
                 &base_url,
-                tx,
+                disable_nuser,
+                disable_nsjail,
+                rx,
             )
             .await
         }));
+        //avoid having all the workers start at the same time
+        tokio::time::sleep(std::time::Duration::from_millis(
+            rng.gen_range(0..sleep_queue * num_workers as u64),
+        ))
+        .await;
     }
     futures::future::try_join_all(handles).await?;
     Ok(())
@@ -340,3 +293,24 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
     let _ = tx.send(());
     Ok(())
 }
+
+pub async fn serve_metrics(
+    addr: SocketAddr,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+) -> Result<(), hyper::Error> {
+    axum::Server::bind(&addr)
+        .serve(
+            Router::new()
+                .route("/metrics", get(metrics))
+                .into_make_service(),
+        )
+        .with_graceful_shutdown(rx.recv().map(drop))
+        .await
+}
+
+async fn metrics() -> Result<String, Error> {
+    let metric_families = prometheus::gather();
+    Ok(prometheus::TextEncoder::new()
+        .encode_to_string(&metric_families)
+        .map_err(anyhow::Error::from)?)
+}

backend/src/main.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -13,7 +14,7 @@ use dotenv::dotenv;
 async fn main() -> anyhow::Result<()> {
     dotenv().ok();
 
-    windmill::initialize_tracing().await?;
+    windmill::initialize_tracing();
 
     let db = windmill::connect_db().await?;
 
@@ -22,6 +23,16 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32);
 
+    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+        .ok()
+        .map(|s| {
+            s.parse::<bool>()
+                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+                .or_else(|_| s.parse::<SocketAddr>().map(Some))
+        })
+        .transpose()?
+        .flatten();
+
     let (server_mode, monitor_mode, migrate_db) = (true, true, true);
 
     if migrate_db {
@@ -29,7 +40,7 @@ async fn main() -> anyhow::Result<()> {
     }
 
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill::shutdown_signal(tx.clone());
+    let shutdown_signal = windmill::shutdown_signal(tx);
 
     if server_mode || monitor_mode || num_workers > 0 {
         let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
@@ -50,7 +61,7 @@ async fn main() -> anyhow::Result<()> {
                         server: "smtp.gmail.com".to_string(),
                         password: std::env::var("SMTP_PASSWORD").unwrap_or("NOPASS".to_string()),
                     },
-                    rx,
+                    rx.resubscribe(),
                 )
                 .await?;
             }
@@ -66,7 +77,20 @@ async fn main() -> anyhow::Result<()> {
                     .ok()
                     .and_then(|x| x.parse::<u64>().ok())
                     .unwrap_or(windmill::DEFAULT_SLEEP_QUEUE);
+                let disable_nuser = std::env::var("DISABLE_NUSER")
+                    .ok()
+                    .and_then(|x| x.parse::<bool>().ok())
+                    .unwrap_or(false);
+                let disable_nsjail = std::env::var("DISABLE_NSJAIL")
+                    .ok()
+                    .and_then(|x| x.parse::<bool>().ok())
+                    .unwrap_or(false);
 
+                tracing::info!(
+                    "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
+                     {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
+                     {timeout}"
+                );
                 windmill::run_workers(
                     db.clone(),
                     addr,
@@ -74,7 +98,9 @@ async fn main() -> anyhow::Result<()> {
                     num_workers,
                     sleep_queue,
                     base_url,
-                    tx.clone(),
+                    disable_nuser,
+                    disable_nsjail,
+                    rx.resubscribe(),
                 )
                 .await?;
             }
@@ -83,12 +109,21 @@ async fn main() -> anyhow::Result<()> {
 
         let monitor_f = async {
             if monitor_mode {
-                windmill::monitor_db(&db, timeout, tx.clone());
+                windmill::monitor_db(&db, timeout, rx.resubscribe());
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f)?;
+        let metrics_f = async {
+            match metrics_addr {
+                Some(addr) => windmill::serve_metrics(addr, rx.resubscribe())
+                    .await
+                    .map_err(anyhow::Error::from),
+                None => Ok(()),
+            }
+        };
+
+        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f, metrics_f)?;
     }
 
     Ok(())

backend/src/parser.rs

@@ -1,592 +1,48 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::collections::HashMap;
-
-use itertools::Itertools;
-use regex::Regex;
 use serde::Serialize;
-use serde_json::json;
 
-use crate::error;
-
-use rustpython_parser::{
-    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
-    parser,
-};
-#[derive(Serialize)]
+#[derive(Serialize, Debug, PartialEq)]
 pub struct MainArgSignature {
     pub star_args: bool,
     pub star_kwargs: bool,
     pub args: Vec<Arg>,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Clone, Debug, PartialEq)]
+#[serde(rename_all(serialize = "lowercase"))]
+pub struct ObjectProperty {
+    pub key: String,
+    pub typ: Box<Typ>,
+}
+
+#[derive(Serialize, Clone, Debug, PartialEq)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum Typ {
-    Str,
+    Str(Option<Vec<String>>),
     Int,
     Float,
     Bool,
-    Dict,
-    List,
+    List(Box<Typ>),
     Bytes,
     Datetime,
+    Resource(String),
+    Email,
+    Sql,
+    Object(Vec<ObjectProperty>),
     Unknown,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Clone, Debug, PartialEq)]
 pub struct Arg {
     pub name: String,
     pub typ: Typ,
     pub default: Option<serde_json::Value>,
     pub has_default: bool,
 }
-
-pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
-    let ast = parser::parse_program(code)
-        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
-        .statements;
-    let param = ast.into_iter().find_map(|x| match x {
-        Located {
-            location: _,
-            node:
-                StatementType::FunctionDef {
-                    is_async: _,
-                    name,
-                    args,
-                    body: _,
-                    decorator_list: _,
-                    returns: _,
-                },
-        } if &name == "main" => Some(*args),
-        _ => None,
-    });
-    if let Some(params) = param {
-        //println!("{:?}", params);
-        let def_arg_start = params.args.len() - params.defaults.len();
-        Ok(MainArgSignature {
-            star_args: params.vararg != Varargs::None,
-            star_kwargs: params.vararg != Varargs::None,
-            args: params
-                .args
-                .into_iter()
-                .enumerate()
-                .map(|(i, x)| {
-                    let default = if i >= def_arg_start {
-                        to_value(&params.defaults[i - def_arg_start].node)
-                    } else {
-                        None
-                    };
-                    Arg {
-                        name: x.arg,
-                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
-                            Located {
-                                location: _,
-                                node: ExpressionType::Identifier { name },
-                            } => match name.as_ref() {
-                                "str" => Typ::Str,
-                                "float" => Typ::Float,
-                                "int" => Typ::Int,
-                                "bool" => Typ::Bool,
-                                "dict" => Typ::Dict,
-                                "list" => Typ::List,
-                                "bytes" => Typ::Bytes,
-                                "datetime" => Typ::Datetime,
-                                "datetime.datetime" => Typ::Datetime,
-                                _ => Typ::Unknown,
-                            },
-                            _ => Typ::Unknown,
-                        }),
-                        has_default: default.is_some(),
-                        default,
-                    }
-                })
-                .collect(),
-        })
-    } else {
-        Err(error::Error::ExecutionErr(
-            "main function was not findable".to_string(),
-        ))
-    }
-}
-
-const STDIMPORTS: [&str; 301] = [
-    "__future__",
-    "_abc",
-    "_aix_support",
-    "_ast",
-    "_asyncio",
-    "_bisect",
-    "_blake2",
-    "_bootsubprocess",
-    "_bz2",
-    "_codecs",
-    "_codecs_cn",
-    "_codecs_hk",
-    "_codecs_iso2022",
-    "_codecs_jp",
-    "_codecs_kr",
-    "_codecs_tw",
-    "_collections",
-    "_collections_abc",
-    "_compat_pickle",
-    "_compression",
-    "_contextvars",
-    "_crypt",
-    "_csv",
-    "_ctypes",
-    "_curses",
-    "_curses_panel",
-    "_datetime",
-    "_dbm",
-    "_decimal",
-    "_elementtree",
-    "_frozen_importlib",
-    "_frozen_importlib_external",
-    "_functools",
-    "_gdbm",
-    "_hashlib",
-    "_heapq",
-    "_imp",
-    "_io",
-    "_json",
-    "_locale",
-    "_lsprof",
-    "_lzma",
-    "_markupbase",
-    "_md5",
-    "_msi",
-    "_multibytecodec",
-    "_multiprocessing",
-    "_opcode",
-    "_operator",
-    "_osx_support",
-    "_overlapped",
-    "_pickle",
-    "_posixshmem",
-    "_posixsubprocess",
-    "_py_abc",
-    "_pydecimal",
-    "_pyio",
-    "_queue",
-    "_random",
-    "_sha1",
-    "_sha256",
-    "_sha3",
-    "_sha512",
-    "_signal",
-    "_sitebuiltins",
-    "_socket",
-    "_sqlite3",
-    "_sre",
-    "_ssl",
-    "_stat",
-    "_statistics",
-    "_string",
-    "_strptime",
-    "_struct",
-    "_symtable",
-    "_thread",
-    "_threading_local",
-    "_tkinter",
-    "_tracemalloc",
-    "_uuid",
-    "_warnings",
-    "_weakref",
-    "_weakrefset",
-    "_winapi",
-    "_zoneinfo",
-    "abc",
-    "aifc",
-    "antigravity",
-    "argparse",
-    "array",
-    "ast",
-    "asynchat",
-    "asyncio",
-    "asyncore",
-    "atexit",
-    "audioop",
-    "base64",
-    "bdb",
-    "binascii",
-    "binhex",
-    "bisect",
-    "builtins",
-    "bz2",
-    "cProfile",
-    "calendar",
-    "cgi",
-    "cgitb",
-    "chunk",
-    "cmath",
-    "cmd",
-    "code",
-    "codecs",
-    "codeop",
-    "collections",
-    "colorsys",
-    "compileall",
-    "concurrent",
-    "configparser",
-    "contextlib",
-    "contextvars",
-    "copy",
-    "copyreg",
-    "crypt",
-    "csv",
-    "ctypes",
-    "curses",
-    "dataclasses",
-    "datetime",
-    "dbm",
-    "decimal",
-    "difflib",
-    "dis",
-    "distutils",
-    "doctest",
-    "email",
-    "encodings",
-    "ensurepip",
-    "enum",
-    "errno",
-    "faulthandler",
-    "fcntl",
-    "filecmp",
-    "fileinput",
-    "fnmatch",
-    "fractions",
-    "ftplib",
-    "functools",
-    "gc",
-    "genericpath",
-    "getopt",
-    "getpass",
-    "gettext",
-    "glob",
-    "graphlib",
-    "grp",
-    "gzip",
-    "hashlib",
-    "heapq",
-    "hmac",
-    "html",
-    "http",
-    "idlelib",
-    "imaplib",
-    "imghdr",
-    "imp",
-    "importlib",
-    "inspect",
-    "io",
-    "ipaddress",
-    "itertools",
-    "json",
-    "keyword",
-    "lib2to3",
-    "linecache",
-    "locale",
-    "logging",
-    "lzma",
-    "mailbox",
-    "mailcap",
-    "marshal",
-    "math",
-    "mimetypes",
-    "mmap",
-    "modulefinder",
-    "msilib",
-    "msvcrt",
-    "multiprocessing",
-    "netrc",
-    "nis",
-    "nntplib",
-    "nt",
-    "ntpath",
-    "nturl2path",
-    "numbers",
-    "opcode",
-    "operator",
-    "optparse",
-    "os",
-    "ossaudiodev",
-    "pathlib",
-    "pdb",
-    "pickle",
-    "pickletools",
-    "pipes",
-    "pkgutil",
-    "platform",
-    "plistlib",
-    "poplib",
-    "posix",
-    "posixpath",
-    "pprint",
-    "profile",
-    "pstats",
-    "pty",
-    "pwd",
-    "py_compile",
-    "pyclbr",
-    "pydoc",
-    "pydoc_data",
-    "pyexpat",
-    "queue",
-    "quopri",
-    "random",
-    "re",
-    "readline",
-    "reprlib",
-    "resource",
-    "rlcompleter",
-    "runpy",
-    "sched",
-    "secrets",
-    "select",
-    "selectors",
-    "shelve",
-    "shlex",
-    "shutil",
-    "signal",
-    "site",
-    "smtpd",
-    "smtplib",
-    "sndhdr",
-    "socket",
-    "socketserver",
-    "spwd",
-    "sqlite3",
-    "sre_compile",
-    "sre_constants",
-    "sre_parse",
-    "ssl",
-    "stat",
-    "statistics",
-    "string",
-    "stringprep",
-    "struct",
-    "subprocess",
-    "sunau",
-    "symtable",
-    "sys",
-    "sysconfig",
-    "syslog",
-    "tabnanny",
-    "tarfile",
-    "telnetlib",
-    "tempfile",
-    "termios",
-    "textwrap",
-    "this",
-    "threading",
-    "time",
-    "timeit",
-    "tkinter",
-    "token",
-    "tokenize",
-    "trace",
-    "traceback",
-    "tracemalloc",
-    "tty",
-    "turtle",
-    "turtledemo",
-    "types",
-    "typing",
-    "unicodedata",
-    "unittest",
-    "urllib",
-    "uu",
-    "uuid",
-    "venv",
-    "warnings",
-    "wave",
-    "weakref",
-    "webbrowser",
-    "winreg",
-    "winsound",
-    "wsgiref",
-    "xdrlib",
-    "xml",
-    "xmlrpc",
-    "zipapp",
-    "zipfile",
-    "zipimport",
-    "",
-];
-
-fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
-    match et {
-        ExpressionType::String {
-            value: StringGroup::Constant { value },
-        } => Some(json!(value)),
-        ExpressionType::Number { value } => match value {
-            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
-            Number::Float { value } => Some(json!(value)),
-            _ => None,
-        },
-        ExpressionType::True => Some(json!(true)),
-        ExpressionType::False => Some(json!(false)),
-
-        ExpressionType::Dict { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .and_then(|x| to_value(&x.node))
-                        .and_then(|x| match x {
-                            serde_json::Value::String(s) => Some(s),
-                            _ => None,
-                        })
-                        .unwrap_or_else(|| "no_key".to_string());
-                    (key, to_value(&v.node))
-                })
-                .collect::<HashMap<String, _>>();
-            Some(json!(v))
-        }
-        ExpressionType::List { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|x| to_value(&x.node))
-                .collect::<Vec<_>>();
-            Some(json!(v))
-        }
-        ExpressionType::None => Some(json!(null)),
-
-        ExpressionType::Call {
-            function: _,
-            args: _,
-            keywords: _,
-        } => Some(json!("<function call>")),
-
-        _ => None,
-    }
-}
-
-pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
-    let find_requirements = code
-        .lines()
-        .find_position(|x| x.starts_with("#requirements:"));
-    let re = Regex::new(r"^\#(\S+)$").unwrap();
-    if let Some((pos, _)) = find_requirements {
-        let lines = code
-            .lines()
-            .skip(pos + 1)
-            .map_while(|x| {
-                re.captures(x)
-                    .map(|x| x.get(1).unwrap().as_str().to_string())
-            })
-            .collect();
-        Ok(lines)
-    } else {
-        let ast = parser::parse_program(code)
-            .map_err(|e| {
-                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
-            })?
-            .statements;
-        let imports = ast
-            .into_iter()
-            .filter_map(|x| match x {
-                Located { location: _, node } => match node {
-                    StatementType::Import { names } => Some(
-                        names
-                            .into_iter()
-                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
-                            .collect::<Vec<String>>(),
-                    ),
-                    StatementType::ImportFrom {
-                        level: _,
-                        module: Some(mod_),
-                        names: _,
-                    } => Some(vec![mod_
-                        .split('.')
-                        .next()
-                        .unwrap_or("")
-                        .to_string()
-                        .replace("_", "-")]),
-                    _ => None,
-                },
-            })
-            .flatten()
-            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
-            .unique()
-            .collect();
-        Ok(imports)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    // Note this useful idiom: importing names from outer (for mod tests) scope.
-    use super::*;
-
-    #[test]
-    fn test_parse_sig() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-
-def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
-
-	print(f\"Hello World and a warm welcome especially to {name}\")
-	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
-	return {\"len\": len(name), \"splitted\": name.split() }
-
-";
-        println!("{}", serde_json::to_string(&parse_signature(code)?)?);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_imports() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-import wmill
-from zanzibar.estonie import talin
-import matplotlib.pyplot as plt
-
-def main():
-    pass
-
-";
-        let r = parse_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_imports2() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-#requirements:
-#burkina=0.4
-#nigeria
-#
-#congo
-
-import os
-import wmill
-from zanzibar.estonie import talin
-
-def main():
-    pass
-
-";
-        let r = parse_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
-
-        Ok(())
-    }
-}

backend/src/parser_py.rs

@@ -0,0 +1,734 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use std::collections::HashMap;
+
+use itertools::Itertools;
+use phf::phf_map;
+use regex::Regex;
+use serde_json::json;
+
+use crate::{
+    error,
+    parser::{Arg, MainArgSignature, Typ},
+};
+
+use rustpython_parser::{
+    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
+    parser,
+};
+
+fn filter_non_main(code: &str) -> String {
+    const DEF_MAIN: &str = "def main(";
+
+    let mut filtered_code = String::new();
+    let mut code_iter = code.split("\n");
+    let mut remaining: String = String::new();
+    while let Some(line) = code_iter.next() {
+        if line.starts_with(DEF_MAIN) {
+            filtered_code += DEF_MAIN;
+            remaining += line.strip_prefix(DEF_MAIN).unwrap();
+            remaining += &code_iter.join("\n");
+            break;
+        }
+    }
+    if filtered_code.is_empty() {
+        return String::new();
+    }
+    let mut chars = remaining.chars();
+    let mut open_parens = 1;
+
+    while let Some(c) = chars.next() {
+        if c == '(' {
+            open_parens += 1;
+        } else if c == ')' {
+            open_parens -= 1;
+        }
+        filtered_code.push(c);
+        if open_parens == 0 {
+            break;
+        }
+    }
+
+    filtered_code.push_str(": return");
+    return filtered_code;
+}
+
+pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
+    let filtered_code = filter_non_main(code);
+    if filtered_code.is_empty() {
+        return Err(error::Error::BadRequest(
+            "No main function found".to_string(),
+        ));
+    }
+    let ast = parser::parse_program(&filtered_code)
+        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
+        .statements;
+    let param = ast.into_iter().find_map(|x| match x {
+        Located {
+            location: _,
+            node:
+                StatementType::FunctionDef {
+                    is_async: _,
+                    name,
+                    args,
+                    body: _,
+                    decorator_list: _,
+                    returns: _,
+                },
+        } if &name == "main" => Some(*args),
+        _ => None,
+    });
+    if let Some(params) = param {
+        //println!("{:?}", params);
+        let def_arg_start = params.args.len() - params.defaults.len();
+        Ok(MainArgSignature {
+            star_args: params.vararg != Varargs::None,
+            star_kwargs: params.vararg != Varargs::None,
+            args: params
+                .args
+                .into_iter()
+                .enumerate()
+                .map(|(i, x)| {
+                    let default = if i >= def_arg_start {
+                        to_value(&params.defaults[i - def_arg_start].node)
+                    } else {
+                        None
+                    };
+                    Arg {
+                        name: x.arg,
+                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
+                            Located { location: _, node: ExpressionType::Identifier { name } } => {
+                                match name.as_ref() {
+                                    "str" => Typ::Str(None),
+                                    "float" => Typ::Float,
+                                    "int" => Typ::Int,
+                                    "bool" => Typ::Bool,
+                                    "dict" => Typ::Object(vec![]),
+                                    "list" => Typ::List(Box::new(Typ::Str(None))),
+                                    "bytes" => Typ::Bytes,
+                                    "datetime" => Typ::Datetime,
+                                    "datetime.datetime" => Typ::Datetime,
+                                    _ => Typ::Unknown,
+                                }
+                            }
+                            _ => Typ::Unknown,
+                        }),
+                        has_default: default.is_some(),
+                        default,
+                    }
+                })
+                .collect(),
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable".to_string(),
+        ))
+    }
+}
+
+fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
+    match et {
+        ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
+        ExpressionType::Number { value } => match value {
+            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
+            Number::Float { value } => Some(json!(value)),
+            _ => None,
+        },
+        ExpressionType::True => Some(json!(true)),
+        ExpressionType::False => Some(json!(false)),
+
+        ExpressionType::Dict { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = k
+                        .as_ref()
+                        .and_then(|x| to_value(&x.node))
+                        .and_then(|x| match x {
+                            serde_json::Value::String(s) => Some(s),
+                            _ => None,
+                        })
+                        .unwrap_or_else(|| "no_key".to_string());
+                    (key, to_value(&v.node))
+                })
+                .collect::<HashMap<String, _>>();
+            Some(json!(v))
+        }
+        ExpressionType::List { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|x| to_value(&x.node))
+                .collect::<Vec<_>>();
+            Some(json!(v))
+        }
+        ExpressionType::None => Some(json!(null)),
+
+        ExpressionType::Call { function: _, args: _, keywords: _ } => {
+            Some(json!("<function call>"))
+        }
+
+        _ => None,
+    }
+}
+
+static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
+    "psycopg2" => "psycopg2-binary"
+};
+
+fn replace_import(x: String) -> String {
+    PYTHON_IMPORTS_REPLACEMENT
+        .get(&x)
+        .map(|x| x.to_owned())
+        .unwrap_or(&x)
+        .to_string()
+}
+
+pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
+    let find_requirements = code
+        .lines()
+        .find_position(|x| x.starts_with("#requirements:"));
+    let re = Regex::new(r"^\#(\S+)$").unwrap();
+    if let Some((pos, _)) = find_requirements {
+        let lines = code
+            .lines()
+            .skip(pos + 1)
+            .map_while(|x| {
+                re.captures(x)
+                    .map(|x| x.get(1).unwrap().as_str().to_string())
+            })
+            .collect();
+        Ok(lines)
+    } else {
+        let code = &code
+            .split("\n")
+            .filter(|x| x.starts_with("import ") || x.starts_with("from "))
+            .join("\n");
+        let ast = parser::parse_program(code)
+            .map_err(|e| {
+                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+            })?
+            .statements;
+
+        let imports = ast
+            .into_iter()
+            .filter_map(|x| match x {
+                Located { location: _, node } => match node {
+                    StatementType::Import { names } => Some(
+                        names
+                            .into_iter()
+                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
+                            .map(replace_import)
+                            .collect::<Vec<String>>(),
+                    ),
+                    StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
+                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
+
+                        Some(vec![replace_import(imprt)])
+                    }
+                    _ => None,
+                },
+            })
+            .flatten()
+            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+            .unique()
+            .collect();
+
+        Ok(imports)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[test]
+    fn test_parse_python_sig() -> anyhow::Result<()> {
+        let code = "
+
+import os
+
+def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
+
+	print(f\"Hello World and a warm welcome especially to {name}\")
+	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
+	return {\"len\": len(name), \"splitted\": name.split() }
+
+";
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        name: "test1".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "name".to_string(),
+                        typ: Typ::Unknown,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "byte".to_string(),
+                        typ: Typ::Bytes,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_2() -> anyhow::Result<()> {
+        let code = "
+
+import os
+
+def main(test1: str,
+    name: datetime.datetime = datetime.now(),
+    byte: bytes = bytes(1)):
+
+	print(f\"Hello World and a warm welcome especially to {name}\")
+	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
+	return {\"len\": len(name), \"splitted\": name.split() }
+
+";
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        name: "test1".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "name".to_string(),
+                        typ: Typ::Unknown,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "byte".to_string(),
+                        typ: Typ::Bytes,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_3() -> anyhow::Result<()> {
+        let code = "
+
+import os
+
+def main(test1: str,
+    name: datetime.datetime = datetime.now(),
+    byte: bytes = bytes(1)): return
+
+";
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        name: "test1".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "name".to_string(),
+                        typ: Typ::Unknown,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "byte".to_string(),
+                        typ: Typ::Bytes,
+                        default: Some(json!("<function call>")),
+                        has_default: true
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_imports() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+import wmill
+from zanzibar.estonie import talin
+import matplotlib.pyplot as plt
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_imports2() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+#requirements:
+#burkina=0.4
+#nigeria
+#
+#congo
+
+import os
+import wmill
+from zanzibar.estonie import talin
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
+
+        Ok(())
+    }
+}
+
+const STDIMPORTS: [&str; 301] = [
+    "__future__",
+    "_abc",
+    "_aix_support",
+    "_ast",
+    "_asyncio",
+    "_bisect",
+    "_blake2",
+    "_bootsubprocess",
+    "_bz2",
+    "_codecs",
+    "_codecs_cn",
+    "_codecs_hk",
+    "_codecs_iso2022",
+    "_codecs_jp",
+    "_codecs_kr",
+    "_codecs_tw",
+    "_collections",
+    "_collections_abc",
+    "_compat_pickle",
+    "_compression",
+    "_contextvars",
+    "_crypt",
+    "_csv",
+    "_ctypes",
+    "_curses",
+    "_curses_panel",
+    "_datetime",
+    "_dbm",
+    "_decimal",
+    "_elementtree",
+    "_frozen_importlib",
+    "_frozen_importlib_external",
+    "_functools",
+    "_gdbm",
+    "_hashlib",
+    "_heapq",
+    "_imp",
+    "_io",
+    "_json",
+    "_locale",
+    "_lsprof",
+    "_lzma",
+    "_markupbase",
+    "_md5",
+    "_msi",
+    "_multibytecodec",
+    "_multiprocessing",
+    "_opcode",
+    "_operator",
+    "_osx_support",
+    "_overlapped",
+    "_pickle",
+    "_posixshmem",
+    "_posixsubprocess",
+    "_py_abc",
+    "_pydecimal",
+    "_pyio",
+    "_queue",
+    "_random",
+    "_sha1",
+    "_sha256",
+    "_sha3",
+    "_sha512",
+    "_signal",
+    "_sitebuiltins",
+    "_socket",
+    "_sqlite3",
+    "_sre",
+    "_ssl",
+    "_stat",
+    "_statistics",
+    "_string",
+    "_strptime",
+    "_struct",
+    "_symtable",
+    "_thread",
+    "_threading_local",
+    "_tkinter",
+    "_tracemalloc",
+    "_uuid",
+    "_warnings",
+    "_weakref",
+    "_weakrefset",
+    "_winapi",
+    "_zoneinfo",
+    "abc",
+    "aifc",
+    "antigravity",
+    "argparse",
+    "array",
+    "ast",
+    "asynchat",
+    "asyncio",
+    "asyncore",
+    "atexit",
+    "audioop",
+    "base64",
+    "bdb",
+    "binascii",
+    "binhex",
+    "bisect",
+    "builtins",
+    "bz2",
+    "cProfile",
+    "calendar",
+    "cgi",
+    "cgitb",
+    "chunk",
+    "cmath",
+    "cmd",
+    "code",
+    "codecs",
+    "codeop",
+    "collections",
+    "colorsys",
+    "compileall",
+    "concurrent",
+    "configparser",
+    "contextlib",
+    "contextvars",
+    "copy",
+    "copyreg",
+    "crypt",
+    "csv",
+    "ctypes",
+    "curses",
+    "dataclasses",
+    "datetime",
+    "dbm",
+    "decimal",
+    "difflib",
+    "dis",
+    "distutils",
+    "doctest",
+    "email",
+    "encodings",
+    "ensurepip",
+    "enum",
+    "errno",
+    "faulthandler",
+    "fcntl",
+    "filecmp",
+    "fileinput",
+    "fnmatch",
+    "fractions",
+    "ftplib",
+    "functools",
+    "gc",
+    "genericpath",
+    "getopt",
+    "getpass",
+    "gettext",
+    "glob",
+    "graphlib",
+    "grp",
+    "gzip",
+    "hashlib",
+    "heapq",
+    "hmac",
+    "html",
+    "http",
+    "idlelib",
+    "imaplib",
+    "imghdr",
+    "imp",
+    "importlib",
+    "inspect",
+    "io",
+    "ipaddress",
+    "itertools",
+    "json",
+    "keyword",
+    "lib2to3",
+    "linecache",
+    "locale",
+    "logging",
+    "lzma",
+    "mailbox",
+    "mailcap",
+    "marshal",
+    "math",
+    "mimetypes",
+    "mmap",
+    "modulefinder",
+    "msilib",
+    "msvcrt",
+    "multiprocessing",
+    "netrc",
+    "nis",
+    "nntplib",
+    "nt",
+    "ntpath",
+    "nturl2path",
+    "numbers",
+    "opcode",
+    "operator",
+    "optparse",
+    "os",
+    "ossaudiodev",
+    "pathlib",
+    "pdb",
+    "pickle",
+    "pickletools",
+    "pipes",
+    "pkgutil",
+    "platform",
+    "plistlib",
+    "poplib",
+    "posix",
+    "posixpath",
+    "pprint",
+    "profile",
+    "pstats",
+    "pty",
+    "pwd",
+    "py_compile",
+    "pyclbr",
+    "pydoc",
+    "pydoc_data",
+    "pyexpat",
+    "queue",
+    "quopri",
+    "random",
+    "re",
+    "readline",
+    "reprlib",
+    "resource",
+    "rlcompleter",
+    "runpy",
+    "sched",
+    "secrets",
+    "select",
+    "selectors",
+    "shelve",
+    "shlex",
+    "shutil",
+    "signal",
+    "site",
+    "smtpd",
+    "smtplib",
+    "sndhdr",
+    "socket",
+    "socketserver",
+    "spwd",
+    "sqlite3",
+    "sre_compile",
+    "sre_constants",
+    "sre_parse",
+    "ssl",
+    "stat",
+    "statistics",
+    "string",
+    "stringprep",
+    "struct",
+    "subprocess",
+    "sunau",
+    "symtable",
+    "sys",
+    "sysconfig",
+    "syslog",
+    "tabnanny",
+    "tarfile",
+    "telnetlib",
+    "tempfile",
+    "termios",
+    "textwrap",
+    "this",
+    "threading",
+    "time",
+    "timeit",
+    "tkinter",
+    "token",
+    "tokenize",
+    "trace",
+    "traceback",
+    "tracemalloc",
+    "tty",
+    "turtle",
+    "turtledemo",
+    "types",
+    "typing",
+    "unicodedata",
+    "unittest",
+    "urllib",
+    "uu",
+    "uuid",
+    "venv",
+    "warnings",
+    "wave",
+    "weakref",
+    "webbrowser",
+    "winreg",
+    "winsound",
+    "wsgiref",
+    "xdrlib",
+    "xml",
+    "xmlrpc",
+    "zipapp",
+    "zipfile",
+    "zipimport",
+    "",
+];

backend/src/parser_ts.rs

@@ -0,0 +1,435 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    error,
+    js_eval::eval_sync,
+    parser::{Arg, MainArgSignature, ObjectProperty, Typ},
+};
+
+use serde_json::Value;
+use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
+use swc_ecma_ast::{
+    ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
+    ModuleDecl, ModuleItem, Number, ObjectLit, Pat, Str, TsArrayType, TsEntityName, TsKeywordType,
+    TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature, TsType,
+    TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
+};
+use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+
+pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Typescript(TsConfig::default()),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let ast = parser
+        .parse_module()
+        .map_err(|_| {
+            error::Error::ExecutionErr(format!(
+                "Error while parsing code, it is invalid typescript"
+            ))
+        })?
+        .body;
+
+    // println!("{ast:?}");
+    let params = ast.into_iter().find_map(|x| match x {
+        ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
+            decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, function, .. }),
+            ..
+        })) if &sym.to_string() == "main" => Some(function.params),
+        _ => None,
+    });
+    if let Some(params) = params {
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: params
+                .into_iter()
+                .map(|x| match x.pat {
+                    Pat::Ident(ident) => {
+                        let (name, typ, nullable) = binding_ident_to_arg(&ident);
+                        Ok(Arg {
+                            name,
+                            typ,
+                            default: None,
+                            has_default: ident.id.optional || nullable,
+                        })
+                    }
+                    Pat::Assign(AssignPat { left, right, .. }) => {
+                        let (name, mut typ, _nullable) =
+                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
+                                error::Error::ExecutionErr(format!(
+                                    "parameter syntax unsupported: `{}`",
+                                    cm.span_to_snippet(left.span())
+                                        .unwrap_or_else(|_| cm.span_to_string(left.span()))
+                                ))
+                            })?;
+
+                        let span = match *right {
+                            Expr::Lit(Lit::Str(Str { span, .. })) => Some(span),
+                            Expr::Lit(Lit::Num(Number { span, .. })) => Some(span),
+                            Expr::Lit(Lit::BigInt(BigInt { span, .. })) => Some(span),
+                            Expr::Lit(Lit::Bool(Bool { span, .. })) => Some(span),
+                            Expr::Object(ObjectLit { span, .. }) => Some(span),
+                            Expr::Array(ArrayLit { span, .. }) => Some(span),
+                            _ => None,
+                        };
+                        let expr = span
+                            .and_then(|x| cm.span_to_snippet(x).ok())
+                            .map(|x| serde_json::from_str(&x).map_err(|_| x));
+
+                        let default = match expr.clone() {
+                            Some(Ok(x)) => Some(x),
+                            Some(Err(x)) => eval_sync(&x).ok(),
+                            None => None,
+                        };
+
+                        if typ == Typ::Unknown && default.is_some() {
+                            typ = json_to_typ(default.as_ref().unwrap());
+                        }
+                        Ok(Arg { name, typ, default, has_default: true })
+                    }
+                    _ => Err(error::Error::ExecutionErr(format!(
+                        "parameter syntax unsupported: `{}`",
+                        cm.span_to_snippet(x.span())
+                            .unwrap_or_else(|_| cm.span_to_string(x.span()))
+                    ))),
+                })
+                .collect::<Result<Vec<Arg>, error::Error>>()?,
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable (expected to find 'export main function(...)'"
+                .to_string(),
+        ))
+    }
+}
+
+fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
+    let (typ, nullable) = type_ann
+        .as_ref()
+        .map(|x| tstype_to_typ(&*x.type_ann))
+        .unwrap_or((Typ::Unknown, false));
+    (id.sym.to_string(), typ, nullable)
+}
+
+fn json_to_typ(js: &Value) -> Typ {
+    match js {
+        Value::String(_) => Typ::Str(None),
+        Value::Number(n) if n.is_i64() => Typ::Int,
+        Value::Number(_) => Typ::Float,
+        Value::Bool(_) => Typ::Bool,
+        Value::Object(o) => Typ::Object(
+            o.iter()
+                .map(|(k, v)| ObjectProperty { key: k.to_string(), typ: Box::new(json_to_typ(v)) })
+                .collect(),
+        ),
+        Value::Array(a) => Typ::List(Box::new(a.first().map(json_to_typ).unwrap_or(Typ::Unknown))),
+        _ => Typ::Unknown,
+    }
+}
+
+fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
+    //println!("{:?}", ts_type);
+    match ts_type {
+        TsType::TsKeywordType(t) => (
+            match t.kind {
+                TsKeywordTypeKind::TsObjectKeyword => Typ::Object(vec![]),
+                TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
+                TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
+                TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
+                TsKeywordTypeKind::TsStringKeyword => Typ::Str(None),
+                _ => Typ::Unknown,
+            },
+            false,
+        ),
+        TsType::TsTypeLit(TsTypeLit { members, .. }) => {
+            let properties = members
+                .into_iter()
+                .filter_map(|x| match x {
+                    TsTypeElement::TsPropertySignature(TsPropertySignature {
+                        key,
+                        type_ann,
+                        ..
+                    }) => match (*key.to_owned(), type_ann) {
+                        (Expr::Ident(Ident { sym, .. }), type_ann) => Some(ObjectProperty {
+                            key: sym.to_string(),
+                            typ: type_ann
+                                .as_ref()
+                                .map(|typ| Box::new(tstype_to_typ(&*typ.type_ann).0))
+                                .unwrap_or(Box::new(Typ::Unknown)),
+                        }),
+                        _ => None,
+                    },
+                    _ => None,
+                })
+                .collect();
+            (Typ::Object(properties), false)
+        }
+        // TODO: we can do better here and extract the inner type of array
+        TsType::TsArrayType(TsArrayType { elem_type, .. }) => {
+            (Typ::List(Box::new(tstype_to_typ(&**elem_type).0)), false)
+        }
+        TsType::TsLitType(TsLitType { lit: TsLit::Str(Str { value, .. }), .. }) => {
+            (Typ::Str(Some(vec![value.to_string()])), false)
+        }
+        TsType::TsOptionalType(TsOptionalType { type_ann, .. }) => {
+            (tstype_to_typ(type_ann).0, true)
+        }
+        TsType::TsUnionOrIntersectionType(TsUnionOrIntersectionType::TsUnionType(
+            TsUnionType { types, .. },
+        )) => {
+            if let Some(p) = if types.len() != 2 {
+                None
+            } else {
+                types.into_iter().position(|x| match **x {
+                    TsType::TsKeywordType(TsKeywordType { kind, .. }) => {
+                        kind == TsKeywordTypeKind::TsUndefinedKeyword
+                            || kind == TsKeywordTypeKind::TsNullKeyword
+                    }
+                    _ => false,
+                })
+            } {
+                let other_p = if p == 0 { 1 } else { 0 };
+                (tstype_to_typ(&types[other_p]).0, true)
+            } else {
+                let literals = types
+                    .into_iter()
+                    .map(|x| match &**x {
+                        TsType::TsLitType(TsLitType {
+                            lit: TsLit::Str(Str { value, .. }), ..
+                        }) => Some(value.to_string()),
+                        _ => None,
+                    })
+                    .collect::<Vec<_>>();
+                if literals.iter().find(|x| x.is_none()).is_some() {
+                    (Typ::Unknown, false)
+                } else {
+                    (
+                        Typ::Str(Some(literals.into_iter().filter_map(|x| x).collect())),
+                        false,
+                    )
+                }
+            }
+        }
+        TsType::TsTypeRef(TsTypeRef { type_name, type_params, .. }) => {
+            let sym = match type_name {
+                TsEntityName::Ident(Ident { sym, .. }) => sym,
+                TsEntityName::TsQualifiedName(p) => &*p.right.sym,
+            };
+            match sym.to_string().as_str() {
+                "Resource" => (
+                    Typ::Resource(
+                        type_params
+                            .as_ref()
+                            .and_then(|x| {
+                                x.params.get(0).and_then(|y| {
+                                    y.as_ts_lit_type().and_then(|z| {
+                                        z.lit.as_str().map(|a| a.to_owned().value.to_string())
+                                    })
+                                })
+                            })
+                            .unwrap_or_else(|| "unknown".to_string()),
+                    ),
+                    false,
+                ),
+                "Base64" => (Typ::Bytes, false),
+                "Email" => (Typ::Email, false),
+                "Sql" => (Typ::Sql, false),
+                _ => (Typ::Unknown, false),
+            }
+        }
+        _ => (Typ::Unknown, false),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use serde_json::json;
+
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+
+    #[test]
+    fn test_parse_deno_sig() -> anyhow::Result<()> {
+        let code = "
+export function main(test1?: string, test2: string = \"burkina\",
+    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], 
+    email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
+    opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
+    min_object: {a: string, b: number}) {
+    console.log(42)
+}
+";
+        assert_eq!(
+            parse_deno_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        name: "test1".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        name: "test2".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("burkina")),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "test3".to_string(),
+                        typ: Typ::Resource("postgres".to_string()),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "b64".to_string(),
+                        typ: Typ::Bytes,
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "ls".to_string(),
+                        typ: Typ::List(Box::new(Typ::Bytes)),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "email".to_string(),
+                        typ: Typ::Email,
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "literal".to_string(),
+                        typ: Typ::Str(Some(vec!["test".to_string()])),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "literal_union".to_string(),
+                        typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        name: "opt_type".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        name: "opt_type_union".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        name: "opt_type_union_union2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        name: "min_object".to_string(),
+                        typ: Typ::Object(vec![
+                            ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
+                            ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Float) }
+                        ]),
+                        default: None,
+                        has_default: false
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> {
+        let code = "
+export function main(test2 = \"burkina\",
+    bool = true,
+    float = 4.2,
+    int = 42,
+    ls = [\"test\"],
+    min_object = {a: \"test\", b: 42}) {
+    console.log(42)
+}
+";
+        assert_eq!(
+            parse_deno_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        name: "test2".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("burkina")),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "bool".to_string(),
+                        typ: Typ::Bool,
+                        default: Some(json!(true)),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "float".to_string(),
+                        typ: Typ::Float,
+                        default: Some(json!(4.2)),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "int".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(42)),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "ls".to_string(),
+                        typ: Typ::List(Box::new(Typ::Str(None))),
+                        default: Some(json!(["test"])),
+                        has_default: true
+                    },
+                    Arg {
+                        name: "min_object".to_string(),
+                        typ: Typ::Object(vec![
+                            ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
+                            ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Int) }
+                        ]),
+                        default: Some(json!({"a": "test", "b": 42})),
+                        has_default: true
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+}

backend/src/resources.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -26,6 +27,7 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
         .route("/get/*path", get(get_resource))
+        .route("/exists/*path", get(exists_resource))
         .route("/get_value/*path", get(get_resource_value))
         .route("/update/*path", post(update_resource))
         .route("/delete/*path", delete(delete_resource))
@@ -33,6 +35,7 @@ pub fn workspaced_service() -> Router {
         .route("/type/list", get(list_resource_types))
         .route("/type/listnames", get(list_resource_types_names))
         .route("/type/get/:name", get(get_resource_type))
+        .route("/type/exists/:name", get(exists_resource_type))
         .route("/type/update/:name", post(update_resource_type))
         .route("/type/delete/:name", delete(delete_resource_type))
         .route("/type/create", post(create_resource_type))
@@ -67,6 +70,7 @@ pub struct Resource {
     pub description: Option<String>,
     pub resource_type: String,
     pub extra_perms: serde_json::Value,
+    pub is_oauth: bool,
 }
 
 #[derive(Deserialize)]
@@ -75,6 +79,7 @@ pub struct CreateResource {
     pub value: Option<serde_json::Value>,
     pub description: Option<String>,
     pub resource_type: String,
+    pub is_oauth: Option<bool>,
 }
 #[derive(Deserialize)]
 struct EditResource {
@@ -104,6 +109,7 @@ async fn list_resources(
             "description",
             "resource_type",
             "extra_perms",
+            "is_oauth",
         ])
         .order_by("path", true)
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -135,7 +141,8 @@ async fn get_resource(
 
     let resource_o = sqlx::query_as!(
         Resource,
-        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         path.to_owned(),
         &w_id
     )
@@ -147,6 +154,24 @@ async fn get_resource(
     Ok(Json(resource))
 }
 
+async fn exists_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn get_resource_value(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -156,7 +181,8 @@ async fn get_resource_value(
     let mut tx = user_db.begin(&authed).await?;
 
     let value_o = sqlx::query_scalar!(
-        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         path.to_owned(),
         &w_id
     )
@@ -178,13 +204,14 @@ async fn create_resource(
 
     sqlx::query!(
         "INSERT INTO resource
-            (workspace_id, path, value, description, resource_type)
-            VALUES ($1, $2, $3, $4, $5)",
+            (workspace_id, path, value, description, resource_type, is_oauth)
+            VALUES ($1, $2, $3, $4, $5, $6)",
         w_id,
         resource.path,
         resource.value,
         resource.description,
         resource.resource_type,
+        resource.is_oauth.unwrap_or(false)
     )
     .execute(&mut tx)
     .await?;
@@ -259,10 +286,16 @@ async fn update_resource(
     if let Some(ndesc) = ns.description {
         sqlb.set_str("description", ndesc);
     }
+
+    sqlb.returning("path");
+
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
-    sqlx::query(&sql).execute(&mut tx).await?;
+    let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+
+    let npath = crate::utils::not_found_if_none(npath_o, "Resource", path)?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -275,16 +308,21 @@ async fn update_resource(
     .await?;
     tx.commit().await?;
 
-    Ok(format!("resource {} updated (npath: {:?})", path, ns.path))
+    Ok(format!("resource {} updated (npath: {:?})", path, npath))
 }
 
 async fn list_resource_types(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<ResourceType>> {
-    let rows = sqlx::query_as!(ResourceType, "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
-        .fetch_all(&db)
-        .await?;
+    let rows = sqlx::query_as!(
+        ResourceType,
+        "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER \
+         BY name",
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
 
     Ok(Json(rows))
 }
@@ -293,9 +331,13 @@ async fn list_resource_types_names(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
-    let rows = sqlx::query_scalar!("SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
-        .fetch_all(&db)
-        .await?;
+    let rows = sqlx::query_scalar!(
+        "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') \
+         ORDER BY name",
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
 
     Ok(Json(rows))
 }
@@ -309,7 +351,8 @@ async fn get_resource_type(
 
     let resource_type_o = sqlx::query_as!(
         ResourceType,
-        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         &name,
         &w_id
     )
@@ -321,6 +364,22 @@ async fn get_resource_type(
     Ok(Json(resource_type))
 }
 
+async fn exists_resource_type(
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<bool> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn create_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/src/schedule.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -9,15 +10,15 @@ use std::str::FromStr;
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::UserDB,
-    error::{self, JsonResult, Result},
+    db::{UserDB, DB},
+    error::{self, Error, JsonResult, Result},
     jobs::{self, push, JobPayload},
     users::Authed,
-    utils::{get_owner_from_path, Pagination, StripPath},
+    utils::{get_owner_from_path, now_from_db, Pagination, StripPath},
 };
 use axum::{
     extract::{Extension, Path, Query},
-    routing::{get, post},
+    routing::{delete, get, post},
     Json, Router,
 };
 
@@ -30,8 +31,10 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_schedule))
         .route("/get/*path", get(get_schedule))
+        .route("/exists/*path", get(exists_schedule))
         .route("/create", post(create_schedule))
         .route("/update/*path", post(edit_schedule))
+        .route("/delete/*path", delete(delete_schedule))
         .route("/setenabled/*path", post(set_enabled))
 }
 
@@ -62,6 +65,7 @@ pub struct NewSchedule {
     pub script_path: String,
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
+    pub enabled: Option<bool>,
 }
 
 pub async fn push_scheduled_job<'c>(
@@ -72,8 +76,9 @@ pub async fn push_scheduled_job<'c>(
         .map_err(|e| error::Error::BadRequest(e.to_string()))?;
 
     let offset = Duration::minutes(schedule.offset_.into());
+    let now = now_from_db(&mut tx).await?;
     let next = sched
-        .after(&(chrono::Utc::now() - offset + Duration::seconds(1)))
+        .after(&(now - offset + Duration::seconds(1)))
         .next()
         .expect("a schedule should have a next event")
         + offset;
@@ -128,8 +133,12 @@ async fn create_schedule(
     cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    let schedule = sqlx::query_as!(Schedule,
-        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, is_flow, args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",
+    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
+
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
+         is_flow, args, enabled) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
@@ -137,10 +146,12 @@ async fn create_schedule(
         &authed.username,
         ns.script_path,
         ns.is_flow,
-        ns.args
+        ns.args,
+        ns.enabled.unwrap_or(false),
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
 
     audit_log(
         &mut tx,
@@ -161,11 +172,43 @@ async fn create_schedule(
     )
     .await?;
 
-    let tx = push_scheduled_job(tx, schedule).await?;
+    let tx = if ns.enabled.unwrap_or(true) {
+        push_scheduled_job(tx, schedule).await?
+    } else {
+        tx
+    };
     tx.commit().await?;
+
     Ok(ns.path.to_string())
 }
 
+async fn check_flow_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+    is_flow: bool,
+    script_path: &str,
+) -> error::Result<()> {
+    if path != script_path || !is_flow {
+        let exists_flow = sqlx::query_scalar!(
+            "SELECT EXISTS (SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
+            path,
+            w_id
+        )
+        .fetch_one(tx)
+        .await?
+        .unwrap_or(false);
+        if exists_flow {
+            return Err(error::Error::BadConfig(format!(
+                "If a schedule has the same path as or a flow, it must be its primary schedule \
+                 and hence can only trigger it. 
+            However the provided path is: {script_path} and is_flow is: {is_flow}",
+            )));
+        };
+    }
+    Ok(())
+}
+
 #[derive(Deserialize)]
 pub struct EditSchedule {
     pub schedule: String,
@@ -175,9 +218,12 @@ pub struct EditSchedule {
 }
 
 async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> Result<()> {
-    sqlx::query!("DELETE FROM queue WHERE schedule_path = $1", path)
-        .execute(db)
-        .await?;
+    sqlx::query!(
+        "DELETE FROM queue WHERE schedule_path = $1 AND running = false",
+        path
+    )
+    .execute(db)
+    .await?;
     Ok(())
 }
 
@@ -193,9 +239,13 @@ async fn edit_schedule(
 
     let mut tx = user_db.begin(&authed).await?;
 
+    check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?;
+
     clear_schedule(&mut tx, path).await?;
-    let schedule = sqlx::query_as!(Schedule,
-        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path \
+         = $5 AND workspace_id = $6 RETURNING *",
         es.schedule,
         es.script_path,
         es.is_flow,
@@ -204,7 +254,8 @@ async fn edit_schedule(
         w_id,
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
 
     if schedule.enabled {
         tx = push_scheduled_job(tx, schedule).await?;
@@ -284,6 +335,24 @@ async fn get_schedule(
     Ok(Json(schedule))
 }
 
+async fn exists_schedule(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 #[derive(Deserialize)]
 pub struct PreviewPayload {
     pub schedule: String,
@@ -355,6 +424,38 @@ pub async fn set_enabled(
     ))
 }
 
+async fn delete_schedule(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(format!("schedule {} deleted", path))
+}
+
 fn schedule_to_user(path: &str) -> String {
     format!("schedule-{}", path.replace('/', "-"))
 }

backend/src/scripts.rs

@@ -1,23 +1,25 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
+use reqwest::Client;
 use serde::Deserializer;
 use sql_builder::prelude::*;
 
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{Error, JsonResult, Result},
-    jobs, parser,
+    error::{to_anyhow, Error, JsonResult, Result},
+    jobs, parser, parser_py, parser_ts,
     users::{owner_to_token_owner, truncate_token, Authed, Tokened},
-    utils::{require_admin, Pagination, StripPath},
+    utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
 };
 use axum::{
-    extract::{Extension, Path, Query},
+    extract::{Extension, Host, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -35,7 +37,14 @@ use std::{
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
 pub fn global_service() -> Router {
-    Router::new().route("/tojsonschema", post(parse_code_to_jsonschema))
+    Router::new()
+        .route(
+            "/python/tojsonschema",
+            post(parse_python_code_to_jsonschema),
+        )
+        .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema))
+        .route("/hub/list", get(list_hub_scripts))
+        .route("/hub/get/*path", get(get_hub_script_by_path))
 }
 
 pub fn workspaced_service() -> Router {
@@ -44,12 +53,32 @@ pub fn workspaced_service() -> Router {
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
         .route("/get/p/*path", get(get_script_by_path))
+        .route("/raw/p/*path", get(raw_script_by_path))
+        .route("/exists/p/*path", get(exists_script_by_path))
         .route("/archive/h/:hash", post(archive_script_by_hash))
         .route("/delete/h/:hash", post(delete_script_by_hash))
         .route("/get/h/:hash", get(get_script_by_hash))
+        .route("/raw/h/:hash", get(raw_script_by_hash))
         .route("/deployment_status/h/:hash", get(get_deployment_status))
 }
 
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
+#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum ScriptLang {
+    Deno,
+    Python3,
+}
+
+impl ScriptLang {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ScriptLang::Deno => "deno",
+            ScriptLang::Python3 => "python3",
+        }
+    }
+}
+
 #[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
@@ -113,6 +142,8 @@ pub struct Script {
     pub extra_perms: serde_json::Value,
     pub lock: Option<String>,
     pub lock_error_logs: Option<String>,
+    pub language: ScriptLang,
+    pub is_trigger: bool,
 }
 
 #[derive(Serialize, Deserialize, sqlx::Type, Debug)]
@@ -138,6 +169,8 @@ pub struct NewScript {
     pub schema: Option<Schema>,
     pub is_template: Option<bool>,
     pub lock: Option<Vec<String>>,
+    pub language: ScriptLang,
+    pub is_trigger: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -152,6 +185,7 @@ pub struct ListScriptQuery {
     pub order_by: Option<String>,
     pub order_desc: Option<bool>,
     pub is_template: Option<bool>,
+    pub is_trigger: Option<bool>,
 }
 
 async fn list_scripts(
@@ -175,12 +209,14 @@ async fn list_scripts(
             "created_by",
             "created_at",
             "archived",
-            "schema",
+            "null as schema",
             "deleted",
             "is_template",
             "extra_perms",
             "null as lock",
             "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
+            "language",
+            "is_trigger",
         ])
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -218,6 +254,9 @@ async fn list_scripts(
     if let Some(it) = &lq.is_template {
         sqlb.and_where_eq("is_template", it);
     }
+    if let Some(it) = &lq.is_trigger {
+        sqlb.and_where_eq("is_trigger", it);
+    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
@@ -226,6 +265,22 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
+async fn list_hub_scripts(
+    Authed { email, username, .. }: Authed,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let asks = list_elems_from_hub(
+        http_client,
+        "https://hub.windmill.dev/searchData?approved=true",
+        email,
+        username,
+        host,
+    )
+    .await?;
+    Ok(Json(asks))
+}
+
 fn hash_script(ns: &NewScript) -> i64 {
     let mut dh = DefaultHasher::new();
     ns.hash(&mut dh);
@@ -298,7 +353,7 @@ async fn create_script(
                 if let Some(clashing_hash) = clashing_hash_o {
                     return Err(Error::BadRequest(format!(
                         "A script with hash {} with same parent_hash has been found. However, the \
-                     lineage must be linear: no 2 scripts can have the same parent",
+                         lineage must be linear: no 2 scripts can have the same parent",
                         ScriptHash(clashing_hash)
                     )));
                 };
@@ -344,10 +399,16 @@ async fn create_script(
         .map(|v| v.1.clone())
         .unwrap_or(json!({}));
 
+    let lock = if ns.language == ScriptLang::Deno {
+        Some("".to_string())
+    } else {
+        ns.lock.as_ref().map(|x| x.join("\n"))
+    };
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
-        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \
-         created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
+        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
+         content, created_by, schema, is_template, extra_perms, lock, language, is_trigger) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
         &w_id,
         &hash.0,
         ns.path,
@@ -359,13 +420,15 @@ async fn create_script(
         ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
         ns.is_template.unwrap_or(false),
         extra_perms,
-        ns.lock.as_ref().map(|x| x.join("\n"))
+        lock,
+        ns.language: ScriptLang,
+        ns.is_trigger.unwrap_or(false),
     )
     .execute(&mut tx)
     .await?;
 
-    let mut tx = if ns.lock.is_none() {
-        let dependencies = parser::parse_imports(&ns.content)?;
+    let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
+        let dependencies = parser_py::parse_python_imports(&ns.content)?;
         let (_, tx) = jobs::push(
             tx,
             &w_id,
@@ -426,6 +489,32 @@ async fn create_script(
     Ok((StatusCode::CREATED, format!("{}", hash)))
 }
 
+pub async fn get_hub_script_by_path(
+    Authed { email, username, .. }: Authed,
+    Path(path): Path<StripPath>,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> Result<String> {
+    let path = path
+        .to_path()
+        .strip_prefix("hub/")
+        .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
+
+    let content = http_get_from_hub(
+        http_client,
+        &format!("https://hub.windmill.dev/raw/{path}.ts"),
+        email,
+        username,
+        host,
+        true,
+    )
+    .await?
+    .text()
+    .await
+    .map_err(to_anyhow)?;
+    Ok(content)
+}
+
 async fn get_script_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -435,8 +524,10 @@ async fn get_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND (workspace_id = $2 OR workspace_id = 'starter'))",
+        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
+         AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
+         (workspace_id = $2 OR workspace_id = 'starter'))",
     )
     .bind(path)
     .bind(w_id)
@@ -448,6 +539,53 @@ async fn get_script_by_path(
     Ok(Json(script))
 }
 
+async fn raw_script_by_path(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path
+        .to_path()
+        .strip_suffix(".ts")
+        .ok_or_else(|| Error::BadRequest("Raw script path must end with .ts".to_string()))?;
+    let mut tx = user_db.begin(&authed).await?;
+
+    let content_o = sqlx::query_scalar!(
+        "SELECT content FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
+         AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
+         (workspace_id = $2 OR workspace_id = 'starter'))",
+         path, w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let content = crate::utils::not_found_if_none(content_o, "Script", path)?;
+    Ok(content)
+}
+
+async fn exists_script_by_path(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND (workspace_id = $2 OR \
+         workspace_id = 'starter') AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 \
+         OR workspace_id = 'starter')))",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn get_script_by_hash_internal<'c>(
     db: &mut Transaction<'c, Postgres>,
     workspace_id: &str,
@@ -477,6 +615,21 @@ async fn get_script_by_hash(
     Ok(Json(r))
 }
 
+async fn raw_script_by_hash(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, hash_str)): Path<(String, String)>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+    let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| {
+        Error::BadRequest("Raw script path must end with .ts".to_string())
+    })?)?);
+    let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
+    tx.commit().await?;
+
+    Ok(r.content)
+}
+
 #[derive(FromRow, Serialize)]
 struct DeploymentStatus {
     lock: Option<String>,
@@ -488,8 +641,10 @@ async fn get_deployment_status(
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<DeploymentStatus> {
     let mut tx = user_db.begin(&authed).await?;
-    let status_o: Option<DeploymentStatus> = sqlx::query_as!(DeploymentStatus,
-        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+    let status_o: Option<DeploymentStatus> = sqlx::query_as!(
+        DeploymentStatus,
+        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR \
+         workspace_id = 'starter')",
         hash.0,
         w_id,
     )
@@ -517,7 +672,8 @@ async fn archive_script_by_path(
         &w_id
     )
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -545,7 +701,9 @@ async fn archive_script_by_hash(
     )
     .bind(&hash.0)
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -571,13 +729,15 @@ async fn delete_script_by_hash(
 
     require_admin(authed.is_admin, &authed.username)?;
     let script = sqlx::query_as::<_, Script>(
-        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND workspace_id = $2\
-         RETURNING *",
+        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND \
+         workspace_id = $2RETURNING *",
     )
     .bind(&hash.0)
     .bind(&w_id)
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("deleting script by hash {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -593,10 +753,16 @@ async fn delete_script_by_hash(
     Ok(Json(script))
 }
 
-async fn parse_code_to_jsonschema(
+async fn parse_python_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<parser::MainArgSignature> {
-    parser::parse_signature(&code).map(Json)
+    parser_py::parse_python_signature(&code).map(Json)
+}
+
+async fn parse_deno_code_to_jsonschema(
+    Json(code): Json<String>,
+) -> JsonResult<parser::MainArgSignature> {
+    parser_ts::parse_deno_signature(&code).map(Json)
 }
 
 pub fn to_i64(s: &str) -> Result<i64> {

backend/src/static_assets.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -11,6 +12,7 @@ use axum::{
     response::IntoResponse,
 };
 
+use mime_guess::mime;
 use rust_embed::RustEmbed;
 
 // static_handler is a handler that serves static files from the
@@ -45,12 +47,22 @@ fn serve_path(path: String) -> Response<BoxBody> {
         Some(content) => {
             let body = body::boxed(body::Full::from(content.data));
             let mime = mime_guess::from_path(path).first_or_octet_stream();
-            Response::builder()
-                .header(header::CONTENT_TYPE, mime.as_ref())
-                .header(header::CACHE_CONTROL, "max-age=3600".to_owned())
-                .body(body)
-                .unwrap()
+            let mut res = Response::builder().header(header::CONTENT_TYPE, mime.as_ref());
+            if mime.as_ref() == mime::APPLICATION_JAVASCRIPT {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else if (mime.type_(), mime.subtype()) == (mime::TEXT, mime::CSS) {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else if (mime.type_()) == (mime::IMAGE) {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else {
+                res = res.header(header::CACHE_CONTROL, "no-cache, no-store, must-revalidate");
+            }
+            res.body(body).unwrap()
         }
+        None if path.as_str().starts_with("_app/") => Response::builder()
+            .status(404)
+            .body(body::boxed(body::Empty::new()))
+            .unwrap(),
         None => serve_path("200.html".to_owned()),
     }
 }

backend/src/tracing_init.rs

@@ -0,0 +1,96 @@
+use ::tracing::{field, Metadata, Span};
+use ::tracing_subscriber::{
+    filter::filter_fn,
+    fmt::{format, Layer},
+    prelude::*,
+    EnvFilter,
+};
+use hyper::Response;
+use tower_http::trace::{MakeSpan, OnResponse};
+
+#[derive(Clone)]
+pub struct MyOnResponse {}
+
+impl<B> OnResponse<B> for MyOnResponse {
+    fn on_response(
+        self,
+        response: &Response<B>,
+        latency: std::time::Duration,
+        _span: &tracing::Span,
+    ) {
+        tracing::info!(
+            latency = latency.as_millis(),
+            status = response.status().as_u16(),
+            "response"
+        )
+    }
+}
+
+#[derive(Clone)]
+pub struct MyMakeSpan {}
+
+impl<B> MakeSpan<B> for MyMakeSpan {
+    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
+        tracing::info_span!(
+            "request",
+            method = %request.method(),
+            uri = %request.uri(),
+            username = field::Empty,
+            workspace_id = field::Empty,
+            email = field::Empty,
+        )
+    }
+}
+
+fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
+    tracing_subscriber::fmt::layer()
+        .json()
+        .flatten_event(true)
+        .with_span_list(false)
+        .with_current_span(true)
+}
+
+fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::Compact>> {
+    tracing_subscriber::fmt::layer().compact()
+}
+
+fn filter_metadata(meta: &Metadata) -> bool {
+    meta.target().starts_with("windmill")
+}
+
+pub fn initialize_tracing() {
+    let tokio_console = std::env::var("TOKIO_CONSOLE")
+        .map(|x| x == "true")
+        .unwrap_or(false);
+    let json_fmt = std::env::var("JSON_FMT")
+        .map(|x| x == "true")
+        .unwrap_or(false);
+
+    let env_filter = EnvFilter::from_default_env();
+
+    let nenv_filter = if tokio_console {
+        env_filter
+            .add_directive("runtime=trace".parse().unwrap())
+            .add_directive("tokio=trace".parse().unwrap())
+    } else {
+        env_filter
+    };
+    let ts_base = tracing_subscriber::registry().with(nenv_filter);
+
+    match (json_fmt, tokio_console) {
+        (true, true) => ts_base
+            .with(json_layer().with_filter(filter_fn(filter_metadata)))
+            .with(console_subscriber::spawn())
+            .init(),
+        (true, false) => ts_base
+            .with(json_layer().with_filter(filter_fn(filter_metadata)))
+            .init(),
+        (false, true) => ts_base
+            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
+            .with(console_subscriber::spawn())
+            .init(),
+        _ => ts_base
+            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
+            .init(),
+    }
+}