@HrushikeshAnandSarangi has signed the CLA in windmill-labs/windmill#8739

.env

@@ -1,2 +1,2 @@
+SITE_URL=localhost
 DB_PASSWORD=changeme
-WM_BASE_URL=windmill.localhost

.github/CODEOWNERS

@@ -1,4 +0,0 @@
-*       @rubenfiszel
-
-/community/ @fatonramadani @rubenfiszel
-/frontend/ @fatonramadani @rubenfiszel

.github/DockerfileBackendTests

@@ -1,62 +0,0 @@
-FROM python:3.10-slim-buster as nsjail
-
-WORKDIR /nsjail
-
-RUN apt-get -y update \
-    && apt-get install -y \
-    bison=2:3.3.* \
-    flex=2.6.* \
-    g++=4:8.3.* \
-    gcc=4:8.3.* \
-    git=1:2.20.* \
-    libprotobuf-dev=3.6.* \
-    libnl-route-3-dev=3.4.* \
-    make=4.2.* \
-    pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*
-
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
-    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
-RUN make
-
-
-FROM rust:slim-buster as builder
-
-RUN apt-get update && apt-get install -y git libssl-dev pkg-config
-
-RUN apt-get -y update \
-    && apt-get install -y \
-    curl lld
-
-ENV SQLX_OFFLINE=true
-
-COPY ./nsjail /nsjail
-
-RUN mkdir -p /frontend/build
-
-RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
-    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
-    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
-    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV TZ=Etc/UTC
-
-ENV PYTHON_VERSION 3.10.4
-
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
-    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
-    && make -j 4 && make install
-
-RUN /usr/local/bin/python3 -m pip install pip-tools
-RUN /usr/local/bin/python3 -m pip install nltk
-RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
-
-COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
-COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
-
-RUN apt-get update \
-    && apt-get install -y postgresql-client

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,27 +1,38 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-title: 'bug:'
-labels: 'bug'
-assignees: 'rubenfiszel'
+title: ''
+labels: ''
+assignees: ''
 
 ---
 
-**Describe the bug** A clear and concise description of what the bug is.
-
-**To Reproduce** Steps to reproduce the behavior:
+**Describe the bug**
+A clear and concise description of what the bug is.
 
+**To Reproduce**
+Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior** A clear and concise description of what you expected to
-happen.
+**Expected behavior**
+A clear and concise description of what you expected to happen.
 
-**Screenshots** If applicable, add screenshots to help explain your problem.
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
 
-**Windmill version** Go on the left menu -> <user> -> User Settings and copy the
-printed version in "Running windmill version (backend): XXX".
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
 
-**Additional context** Add any other context about the problem here.
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,8 +0,0 @@
----
-name: Feature Request
-about: Create a feature request
-title: 'feature: '
-labels: 'feature'
-assignees: 'rubenfiszel'
-
----

.github/change-versions.sh

@@ -5,15 +5,12 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml
-sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill_pg/pyproject.toml
-# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
-sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
+sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
+sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" Pipfile
+sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" Pipfile
 
 sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" backend/Cargo.lock
-
-cd frontend && npm i --package-lock-only

.github/dependabot.yml

@@ -1,39 +0,0 @@
-# Basic set up for three package managers
-
-version: 2
-updates:
-  # Maintain dependencies for GitHub Actions
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  # Maintain dependencies for npm
-  - package-ecosystem: "npm"
-    directory: "/frontend"
-    schedule:
-      interval: "weekly"
-
-  # Maintain dependencies for cargo
-  - package-ecosystem: "cargo"
-    directory: "/backend"
-    schedule:
-      interval: "weekly"
-
-  # Maintain dependencies for Docker
-  - package-ecosystem: "docker"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  # Maintain dependencies for wmill python client
-  - package-ecosystem: "pip"
-    directory: "/python-client/wmill"
-    schedule:
-      interval: "weekly"
-      
-  # Maintain dependencies for wmill_pg python client
-  - package-ecosystem: "pip"
-    directory: "/python-client/wmill_pg"
-    schedule:
-      interval: "weekly"

.github/pull_hub_items.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-RT=$(curl -s https://hub.windmill.dev/resource_types/list | jq -c -r '.[]')
-for item in ${RT[@]}; do
-    name=$(jq -r '.name' <<< "$item")
-    id=$(jq -r '.id' <<< "$item")
-    echo $name $id 
-    body=$(curl -s -H "accept: application/json" https://hub.windmill.dev/resource_types/${id}/${name})
-    jq -r '.resource_type.schema' <<< "$body" > ./tmp
-    description=$(jq -r '.resource_type.description' <<< "$body")
-    description=$(echo -E $description)
-    echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
-    rm ./tmp
-done

.github/workflows/automerge-dependabot.yml

@@ -1,26 +0,0 @@
-name: dependabot auto-merge
-
-on: pull_request_target
-
-permissions:
-  contents: read
-  pull-requests: read
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.3.3
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Enable auto-merge for Dependabot PRs
-        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
-        run: |
-          echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
-          gh pr review --approve "$PR_URL"
-          gh pr merge --auto --squash "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/backend-test.yml

@@ -1,41 +0,0 @@
-name: Backend only integration tests
-
-on:
-  push:
-    branches:
-      - "main"
-    paths:
-      - "backend/**"
-      - ".github/workflows/backend-test.yml"
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths:
-      - "backend/**"
-      - ".github/workflows/backend-test.yml"
-
-jobs:
-  cargo_test:
-    runs-on: [self-hosted, new]
-    container:
-      image: ghcr.io/windmill-labs/backend-tests
-      options: --privileged
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_PASSWORD: changeme
-
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: backend -> target
-      - name: cargo test
-        timeout-minutes: 5
-        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test -- --nocapture

.github/workflows/change-versions.yml

@@ -7,9 +7,8 @@ on:
 jobs:
   change_version:
     runs-on: ubuntu-latest
-    container: node:18
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
       - uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/deno_on_release.yml

@@ -1,49 +0,0 @@
-name: Publish deno-client
-on:
-  push:
-    tags:
-      - "v*"
-env:
-  repo: windmill-deno-client
-
-jobs:
-  build_deno_and_push_to_repo:
-    runs-on: ubuntu-latest
-    container: openapitools/openapi-generator-cli:v6.0.0-beta
-    steps:
-      - uses: actions/checkout@v3
-      - name: generate_deno
-        run: |
-          cd deno-client
-          rm .gitignore
-          ./generate.sh
-      - name: Pushes to another repository
-        id: push_directory
-        uses: cpina/github-action-push-to-another-repository@devel
-        env:
-          API_TOKEN_GITHUB: ${{ secrets.DENO_PAT }}
-        with:
-          source-directory: deno-client/
-          destination-github-username: ${{ github.repository_owner }}
-          destination-repository-name: ${{ env.repo }}
-          user-email: ruben@windmill.dev
-          commit-message: See ORIGIN_COMMIT from $GITHUB_REF
-          target-branch: main
-
-  tag_repo:
-    needs: [build_deno_and_push_to_repo]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: ${{ github.repository_owner }}/${{ env.repo }}
-          token: ${{ secrets.DENO_PAT }}
-          path: ./client
-
-      - name: Push client
-        run: |
-          cd ./client
-          git config --global user.email "ruben@windmill.dev"
-          git config --global user.name "rubenfiszel[bot]"
-          git tag -a ${{ github.ref_name }} -m "${{ github.ref_name }}"
-          git push --tags

.github/workflows/deploy_to_windmill.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [main]
     paths:
-      - "community/**"
+      - "community/*"
 
 jobs:
   deploy:
@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
+        uses: windmill-labs/windmill-gh-action-deploy@v1.0.0
         with:
           dry_run: false
           input_dir: community

.github/workflows/docker-image.yml

@@ -19,7 +19,7 @@ jobs:
   build:
     runs-on: [self-hosted, new]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - name: Set up Docker Buildx
@@ -35,21 +35,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-      - name: Build and push privately
-        uses: docker/build-push-action@v3
-        if: github.event_name == 'pull_request'
-        with:
-          context: .
-          push: true
-          tags: |
-            ${{ steps.metalocal.outputs.tags }}
-
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
-
       - name: Docker meta
-        if: github.event_name != 'pull_request'
         id: meta
         uses: docker/metadata-action@v4
         with:
@@ -68,44 +54,15 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push publicly
+      - name: Build and push
         uses: docker/build-push-action@v3
-        if: github.event_name != 'pull_request'
         with:
           context: .
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
             ${{ steps.metalocal.outputs.tags }}
             ${{ steps.meta.outputs.tags }}
+
           labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
-  playwright:
-    runs-on: [self-hosted, new]
-    needs: [build]
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_USER: admin
-          POSTGRES_PASSWORD: changeme
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v3
-      - name: "Docker"
-        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-        id: docker-container
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
-      - name: "Clean up"
-        run: docker kill ${{ steps.docker-container.outputs.id }}
-        if: always()
+          cache-from: type=registry,ref=registry.wimill.xyz/windmilllabs/windmill:buildcache
+          cache-to: type=registry,ref=registry.wimill.xyz/windmilllabs/windmill:buildcache,mode=max

.github/workflows/lsp_on_release.yml

@@ -0,0 +1,37 @@
+name: Publish LSP Server
+on:
+  push:
+    branches: [main]
+    paths:
+      - "python-client/*W"
+      - "lsp/*"
+    tags:
+      - "*"
+jobs:
+  build:
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker meta local
+        id: metalocal
+        uses: docker/metadata-action@v4
+        with:
+          images: registry.wimill.xyz/windmilllabs/lsp
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:lsp"
+          push: true
+          tags: ${{ steps.metalocal.outputs.tags }}
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=registry.wimill.xyz/windmilllabs/lsp:buildcache
+          cache-to: type=registry,ref=registry.wimill.xyz/windmilllabs/lsp:buildcache,mode=max

.github/workflows/pull-hub.yml

@@ -1,20 +0,0 @@
-name: Pull Hub Items
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 */1 * *"
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  change_version:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Pull hub
-        run: ./.github/pull_hub_items.sh
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        with:
-          title: sync hub items with community
-          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -1,19 +1,14 @@
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}-lsp
-
 name: Publish python-client
 on:
   push:
     tags:
       - "v*"
-  workflow_dispatch:
 
 jobs:
-  publish_pypi:
+  build_lsp:
     runs-on: [self-hosted, new]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Upload python client
         env:
           PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
@@ -22,44 +17,3 @@ jobs:
           export PATH=$PATH:/usr/local/bin
           export PATH=$PATH:/root/.local/bin
           ./publish.sh
-
-  publish_lsp:
-    needs: [publish_pypi]
-    runs-on: [self-hosted, new]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Login to registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push publicly
-        uses: docker/build-push-action@v3
-        with:
-          context: "{{defaultContext}}:lsp"
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.metalocal.outputs.tags }}
-            ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/release-please.yml

@@ -8,7 +8,7 @@ jobs:
     name: "Release please"
     runs-on: ubuntu-latest
     steps:
-      - uses: GoogleCloudPlatform/release-please-action@v3
+      - uses: GoogleCloudPlatform/release-please-action@v2
         with:
           release-type: simple
           package-name: windmill

.github/workflows/sign-cla.yml

@@ -12,7 +12,7 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: cla-assistant/github-action@v2.2.1
+        uses: cla-assistant/github-action@v2.1.3-beta
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}

CHANGELOG.md

@@ -1,622 +1,3 @@
 # Changelog
 
-
-## [1.32.0](https://github.com/windmill-labs/windmill/compare/v1.31.0...v1.32.0) (2022-08-21)
-
-
-### Features
-
-* **backend:** failure_module ([#452](https://github.com/windmill-labs/windmill/issues/452)) ([32d067f](https://github.com/windmill-labs/windmill/commit/32d067f8c078fd7940c2c4bab8dbb01de876503e))
-* **frontend:** Open/Close UI ([#445](https://github.com/windmill-labs/windmill/issues/445)) ([7e4aac9](https://github.com/windmill-labs/windmill/commit/7e4aac997175bf2ba479021742e5aa8abab4ff41))
-* private imports ([a5343fa](https://github.com/windmill-labs/windmill/commit/a5343fa959a237120fc22d6a3c06da3b29a3f990))
-* rely on PG time rather than worker time ([0057266](https://github.com/windmill-labs/windmill/commit/00572668f16183f7508b9966213cbcc9c106da51))
-
-
-### Bug Fixes
-
-* **backend:** clear_schedule only clear non running jobs ([0cd814c](https://github.com/windmill-labs/windmill/commit/0cd814cfec3ab088f7646b6b9f6970e48961e710))
-* **backend:** fixes forloop with 257 items only iterates once ([#446](https://github.com/windmill-labs/windmill/issues/446)) ([bae8573](https://github.com/windmill-labs/windmill/commit/bae85732ff7c70796c2defcd0430d64dedeb36f7))
-* **backend:** started_at info for completed_job is no more completed_at ([77a6851](https://github.com/windmill-labs/windmill/commit/77a685144ddc65c8e5205688ce7e411a14f7915b))
-* cancel a flow now does the expected behavior ([c0e9cd0](https://github.com/windmill-labs/windmill/commit/c0e9cd05641d28336cc26eee5167a397149d61f2))
-* **deno-client:** pg module now supports prepared statements ([5900a03](https://github.com/windmill-labs/windmill/commit/5900a03c045861732bbf6f7bff1280f3c94b86ce))
-* **deno-client:** wrap the deno-postgres client and not the query statement ([68aaf32](https://github.com/windmill-labs/windmill/commit/68aaf3267ce183e366696ebadc644580976ed7ce))
-* **frontend:** Fix loops pickable properties ([#441](https://github.com/windmill-labs/windmill/issues/441)) ([0681472](https://github.com/windmill-labs/windmill/commit/068147251c831d3ab8564ccb909ad72ef2e32e74))
-* **frontend:** input checks refresh when schema change ([15f7cad](https://github.com/windmill-labs/windmill/commit/15f7cadc3d179993b70e1f7584d532528aaabb52))
-* **frontend:** link to schedule in runs discriminate isFlows ([7d76e69](https://github.com/windmill-labs/windmill/commit/7d76e69be9753cc572ce7c085d0191a31471d9e9))
-* **frontend:** simplify flow preview  logic([#450](https://github.com/windmill-labs/windmill/issues/450)) ([bc5a568](https://github.com/windmill-labs/windmill/commit/bc5a5688ce9c351ad745be225c11a977c1ad2afb))
-* handle 0 length for-loops in the backend ([#440](https://github.com/windmill-labs/windmill/issues/440)) ([561e13e](https://github.com/windmill-labs/windmill/commit/561e13e51ee7ffcf20bc524c22d756ea582d546e))
-* restart zombie jobs was restarting all jobs ([da77d04](https://github.com/windmill-labs/windmill/commit/da77d040942c01b0011e76546dddd6aaa7786b8f))
-
-## [1.31.0](https://github.com/windmill-labs/windmill/compare/v1.30.0...v1.31.0) (2022-08-17)
-
-
-### Features
-
-* allow to configure port via envar ([#407](https://github.com/windmill-labs/windmill/issues/407)) ([34be056](https://github.com/windmill-labs/windmill/commit/34be0564f89f942478c25e77fd77a515367a6afd))
-* db users: admin -> windmill_admin, app -> windmill_user ([#404](https://github.com/windmill-labs/windmill/issues/404)) ([1c40f01](https://github.com/windmill-labs/windmill/commit/1c40f01e5d8e3d854de4c30d9f5e4f731c220ce2))
-* **frontend:** Redesign of the Flow Editor + Arbitrary forloop ([127b0b4](https://github.com/windmill-labs/windmill/commit/127b0b4e5e6a96f91d7e8234cc52d887afb637b0))
-
-
-### Bug Fixes
-
-* **backend:** collecting result when for loop is not the last step [#422](https://github.com/windmill-labs/windmill/issues/422)  ([e606118](https://github.com/windmill-labs/windmill/commit/e6061189438fb3a7e630d2e390075fc3eded984c))
-* **self-hosting:** add lsp and caddy to docke-compose ([#432](https://github.com/windmill-labs/windmill/issues/432)) ([1004518](https://github.com/windmill-labs/windmill/commit/100451878c26d2fa324c6195838accae959a5310))
-* set secure only for https ([1275f5f](https://github.com/windmill-labs/windmill/commit/1275f5f7fb65e32a17d7d397d43d0b49ecd5cd0e))
-* users privileges ([2bdb617](https://github.com/windmill-labs/windmill/commit/2bdb617b1f80104bd3314656603dccb0021e05cb))
-
-## [1.30.0](https://github.com/windmill-labs/windmill/compare/v1.29.0...v1.30.0) (2022-08-13)
-
-
-### Features
-
-* add literal object type support ([#401](https://github.com/windmill-labs/windmill/issues/401)) ([845de82](https://github.com/windmill-labs/windmill/commit/845de8206214ed265aef895f0d13636e6e0e26ce))
-* support union type will null | undefined ([#400](https://github.com/windmill-labs/windmill/issues/400)) ([0384727](https://github.com/windmill-labs/windmill/commit/0384727a56347aa01a5fee06c82bd49eab2522fa))
-* support union types ([#398](https://github.com/windmill-labs/windmill/issues/398)) ([e68ea1b](https://github.com/windmill-labs/windmill/commit/e68ea1b8fc4f88e587121387ecac6858d04ebae2))
-
-## [1.29.0](https://github.com/windmill-labs/windmill/compare/v1.28.1...v1.29.0) (2022-08-10)
-
-
-### Features
-
-* _value, _index => iter.value, iter.index ([07f4a21](https://github.com/windmill-labs/windmill/commit/07f4a217d0c6b46fd3defaa0242d229a60c69463))
-* remove res1 wrapping ([e76a981](https://github.com/windmill-labs/windmill/commit/e76a9816ee09e59d5c38bf0c19231bac8347148c))
-
-
-### Bug Fixes
-
-* do not skip undefined values ([8b68a87](https://github.com/windmill-labs/windmill/commit/8b68a87c523fe13a9f45786ee0fbb57b10efda13))
-* **python:** not filled field with default <function_call> now call the default function ([33962c4](https://github.com/windmill-labs/windmill/commit/33962c44660fd20173a0ae14b00a66a985dd4fc7))
-* surface new _iterator value ([13b1904](https://github.com/windmill-labs/windmill/commit/13b1904a7ab5a6e7a7c82d2a2806648441759756))
-* update logs even if last new log was < 500ms ([c69621f](https://github.com/windmill-labs/windmill/commit/c69621fa7a9a18223e7854f0824bd6fbcabdfe10))
-
-## [1.28.1](https://github.com/windmill-labs/windmill/compare/v1.28.0...v1.28.1) (2022-08-05)
-
-
-### Bug Fixes
-
-* **frontend:** add toggl connect ([#341](https://github.com/windmill-labs/windmill/issues/341)) ([b94895f](https://github.com/windmill-labs/windmill/commit/b94895f24eb4ba1b67f499a98c6e6e8d9d006b14))
-* **frontend:** schedule args in flow ([#343](https://github.com/windmill-labs/windmill/issues/343)) ([350a25c](https://github.com/windmill-labs/windmill/commit/350a25c837b1367fa5568dd1de0196202d632bd0))
-* improve flow viewer with retrieving hub script ([80e28db](https://github.com/windmill-labs/windmill/commit/80e28dbba3e77154c0017bd8e74d144e6aae13fb))
-
-## [1.28.0](https://github.com/windmill-labs/windmill/compare/v1.27.2...v1.28.0) (2022-08-04)
-
-
-### Features
-
-* **frontend:** global flow preview ([#329](https://github.com/windmill-labs/windmill/issues/329)) ([615f69e](https://github.com/windmill-labs/windmill/commit/615f69e935e9c9c0b60edfb6dc2e82aebba623b9))
-
-
-### Bug Fixes
-
-* **api:** add discord webhook manual instructions ([a9a4b9b](https://github.com/windmill-labs/windmill/commit/a9a4b9b21d7b68a3e46c28ce13986d7a9ebd2cac))
-* **backend:** generalize oauth clients to take in extra params ([6332910](https://github.com/windmill-labs/windmill/commit/6332910dd27f78d555f0ab040545e98dedbea89d))
-* **backend:** handle better some flow edge-cases ([3bcd542](https://github.com/windmill-labs/windmill/commit/3bcd542130bc0cb45dfb1fa7681dd4b7beb95c7e))
-* **backend:** handle better some flow edge-cases ([9885361](https://github.com/windmill-labs/windmill/commit/988536128bd04dab94cc686bc2db547e57894587))
-* **backend:** handle better some flow edge-cases ([70de6e3](https://github.com/windmill-labs/windmill/commit/70de6e3972af81aec68b706dca93e16182a584bb))
-* **backend:** prometheus histogram for worker job timer ([#312](https://github.com/windmill-labs/windmill/issues/312)) ([4055586](https://github.com/windmill-labs/windmill/commit/40555868e6221620beca85ebafad2da67e56ec08))
-* **frontend:** add jpeg support ([0e8552b](https://github.com/windmill-labs/windmill/commit/0e8552ba800f13add6b25a83a765dace8d4369e7))
-* **frontend:** loading template pick the language as well ([82c7ddc](https://github.com/windmill-labs/windmill/commit/82c7ddc00e79a1cc5336a0a219f46d705c2c8d88))
-* **frontend:** Use the bracket notation when an identifier is not a valid JS expression ([#327](https://github.com/windmill-labs/windmill/issues/327)) ([05324bd](https://github.com/windmill-labs/windmill/commit/05324bd3562f6066cdc12d74c87033325d1c7ef1))
-* **oauth2:** remove discord oauth integration ([986e76d](https://github.com/windmill-labs/windmill/commit/986e76dc8729a53d09cd83531d474f9b5fe88f35))
-
-## [1.27.2](https://github.com/windmill-labs/windmill/compare/v1.27.1...v1.27.2) (2022-08-02)
-
-
-### Bug Fixes
-
-* **deno-client:** getResource can now fetch non-object values ([b128388](https://github.com/windmill-labs/windmill/commit/b128388cc652d4cd369a88b93985a2c051003abd))
-
-## [1.27.1](https://github.com/windmill-labs/windmill/compare/v1.27.0...v1.27.1) (2022-08-02)
-
-
-### Bug Fixes
-
-* migrate to new style radio button ([893ee94](https://github.com/windmill-labs/windmill/commit/893ee941d72a7036f0ea272c49bbe5cd3eee64d5))
-
-## [1.27.0](https://github.com/windmill-labs/windmill/compare/v1.26.3...v1.27.0) (2022-08-02)
-
-
-### Features
-
-* add primitive sql format ([#320](https://github.com/windmill-labs/windmill/issues/320)) ([9daff2a](https://github.com/windmill-labs/windmill/commit/9daff2a228791234a3dd70c0ee829e284daf1592))
-
-
-### Bug Fixes
-
-* prefer `COPY` over `ADD` ([#319](https://github.com/windmill-labs/windmill/issues/319)) ([24a7e46](https://github.com/windmill-labs/windmill/commit/24a7e46fe99d5a1f7d5b22334fa5f6ce76e82d94))
-* typos ([#301](https://github.com/windmill-labs/windmill/issues/301)) ([9e84b45](https://github.com/windmill-labs/windmill/commit/9e84b458b139e86eb51dba9c5b228f141ca649b3))
-
-## [1.26.3](https://github.com/windmill-labs/windmill/compare/v1.26.2...v1.26.3) (2022-08-01)
-
-
-### Bug Fixes
-
-* displaying which group you are a member of that gave you access to item ([1bd0269](https://github.com/windmill-labs/windmill/commit/1bd026924b8a3b01f7729b627f939d8af872a483))
-* refresh jobs result when hopping from flow to flow ([c86abe6](https://github.com/windmill-labs/windmill/commit/c86abe6ae01efd519f67ead233ebddf39f1539c0))
-
-## [1.26.2](https://github.com/windmill-labs/windmill/compare/v1.26.1...v1.26.2) (2022-07-31)
-
-
-### Bug Fixes
-
-* deno api generator now supports openflow ([5b548a0](https://github.com/windmill-labs/windmill/commit/5b548a0e71669aad90343e70f3f1c9dc3a6d4baf))
-
-## [1.26.1](https://github.com/windmill-labs/windmill/compare/v1.26.0...v1.26.1) (2022-07-31)
-
-
-### Bug Fixes
-
-* encoding state now supports unicode including emojis ([6b61227](https://github.com/windmill-labs/windmill/commit/6b61227481422fe52384f6de8146388a8471ff60))
-
-## [1.26.0](https://github.com/windmill-labs/windmill/compare/v1.25.0...v1.26.0) (2022-07-29)
-
-
-### Features
-
-* resource type picker in schema modal + proper initialization of raw javascript editor when applicable ([01bb107](https://github.com/windmill-labs/windmill/commit/01bb107a0f3e3899ec99718974b2484ab5978c92))
-
-
-### Bug Fixes
-
-* forloop flows unsoundness fix part I ([1b5ce32](https://github.com/windmill-labs/windmill/commit/1b5ce3243b364d02903072a9af5e15447622e9fb))
-* small bar mode and editor nits ([4e3a02a](https://github.com/windmill-labs/windmill/commit/4e3a02a8e44e25e6b5402f732b9af6969d06dcc0))
-
-## [1.25.0](https://github.com/windmill-labs/windmill/compare/v1.24.2...v1.25.0) (2022-07-29)
-
-
-### Features
-
-* base64 support in schema editor ([2cb6e6e](https://github.com/windmill-labs/windmill/commit/2cb6e6e7021819a9aa9618436abf2f0fa5b3587b))
-
-
-### Bug Fixes
-
-* update variable and resources now return error if nothing was updated ([0faabdb](https://github.com/windmill-labs/windmill/commit/0faabdbc40b049258b074c6c20c1406ca14b8481))
-
-## [1.24.2](https://github.com/windmill-labs/windmill/compare/v1.24.1...v1.24.2) (2022-07-28)
-
-
-### Bug Fixes
-
-* get_variable refresh_token bug ([390e9b3](https://github.com/windmill-labs/windmill/commit/390e9b37fb201242ac6983c145c9de5b242f7a7b))
-* if :path is not a valid path, do not even attempt to fetch it ([6dec447](https://github.com/windmill-labs/windmill/commit/6dec4479537164fe17bea7f88fd60b1d4f42e887))
-* monaco editor fixes ([f255cc2](https://github.com/windmill-labs/windmill/commit/f255cc253fcf14850442e8d4bf64635287b88314))
-
-## [1.24.1](https://github.com/windmill-labs/windmill/compare/v1.24.0...v1.24.1) (2022-07-27)
-
-
-### Bug Fixes
-
-* encrypt the refresh token ([a051c21](https://github.com/windmill-labs/windmill/commit/a051c2121a63983f6925ce2e3a9b9deb01df2f04))
-* keep previous refresh token if no new ones were provided ([3feef73](https://github.com/windmill-labs/windmill/commit/3feef738dc145603576649a91f0ddc0e82215841))
-* skip_failures is boolean not bool ([4ca71c1](https://github.com/windmill-labs/windmill/commit/4ca71c1e5da0132724ab4c9771f5fdc590b866f8))
-
-## [1.24.0](https://github.com/windmill-labs/windmill/compare/v1.23.0...v1.24.0) (2022-07-27)
-
-
-### Features
-
-* Add flow input and current step in the prop picker ([#236](https://github.com/windmill-labs/windmill/issues/236)) ([6fbeeae](https://github.com/windmill-labs/windmill/commit/6fbeeae84a207be46490361788dad12918c37c4e))
-* add google login v1 ([fc918a2](https://github.com/windmill-labs/windmill/commit/fc918a24ccf0ad19b81a3ebf630d0f04b56094c8))
-* add schedule settable from pull flows ([caecbfd](https://github.com/windmill-labs/windmill/commit/caecbfd0d9eaadc38372ce7238ed6d3baf9ba6e3))
-* prop picker functional for pull flows ([010acfe](https://github.com/windmill-labs/windmill/commit/010acfe7e365a838078f1a989b54f1539c8bf2e6))
-* skip failures loop ([#258](https://github.com/windmill-labs/windmill/issues/258)) ([de3fe69](https://github.com/windmill-labs/windmill/commit/de3fe699089e2a28aa0032a57a9a03f35646b6ef))
-
-
-### Bug Fixes
-
-* audit logs ([ca4bed3](https://github.com/windmill-labs/windmill/commit/ca4bed34a65440cd790cae9cff19f40df22f92b8))
-* **frontend:** badge google logo for login ([cfec7a9](https://github.com/windmill-labs/windmill/commit/cfec7a97b883dbf83bd9d0707bf015c2aaa4e517))
-* **frontend:** badge needs a little right margin ([c846ed7](https://github.com/windmill-labs/windmill/commit/c846ed76c4102335a5a8aabceaa39d6b7906ef5a))
-* **frontend:** display number field in flows ([a232895](https://github.com/windmill-labs/windmill/commit/a23289563deca70269bd73ec50f324db0b6df791))
-* **frontend:** fork script from hub ([43cacc1](https://github.com/windmill-labs/windmill/commit/43cacc1a66b1e2322c0252c9d1ca954e893aaef8))
-* **frontend:** get refresh token for google services ([2f0d8d5](https://github.com/windmill-labs/windmill/commit/2f0d8d5384fb4eea6a6d5e5e48fd242f8d0c40fa))
-* **frontend:** get refresh token for google services ([8dfe688](https://github.com/windmill-labs/windmill/commit/8dfe688a6a2388cecb1460913a25ab49ec297b1b))
-* **frontend:** get refresh token for google services ([a2c5dc1](https://github.com/windmill-labs/windmill/commit/a2c5dc18a38045cbefc7d4b86d786a3c8fcb3ca8))
-* import from JSON load schemas ([88dd7b0](https://github.com/windmill-labs/windmill/commit/88dd7b0abbd1a0469fc949c8045f61ddc304701d))
-* multiple UI fixes ([a334029](https://github.com/windmill-labs/windmill/commit/a33402978720470530baecf51c2d17ecafd13ab0))
-* multiple UI fixes ([904f0f3](https://github.com/windmill-labs/windmill/commit/904f0f3e69034421d524a66e0c4697ff42d89efe))
-
-## [1.23.0](https://github.com/windmill-labs/windmill/compare/v1.22.0...v1.23.0) (2022-07-25)
-
-
-### Features
-
-* add editor bar to inline scripts of flows ([7a6a2c9](https://github.com/windmill-labs/windmill/commit/7a6a2c982daef9aa80e34aa6cbd4889a3c5ec807))
-* **backend:** do not require visibility on job to see job if in possesion of uuid ([b054229](https://github.com/windmill-labs/windmill/commit/b05422963b27d74de8bb6d3be18538d57a71cfe7))
-* **frontend:** deeper integration with the hub ([bb58eba](https://github.com/windmill-labs/windmill/commit/bb58eba2b521aef67b91cfc23f3ddcc8a001e18f))
-* **frontend:** title everywhere ([38987c6](https://github.com/windmill-labs/windmill/commit/38987c6068c4cc2d9accbc368a67362e74adcabf))
-* hub flows integration ([62777b7](https://github.com/windmill-labs/windmill/commit/62777b7a7888b3456f7f864cbb1acd887b172adc))
-
-
-### Bug Fixes
-
-* display websocket status in flow inline editor ([9e9138e](https://github.com/windmill-labs/windmill/commit/9e9138e4eeaea962dbb149ad4c1450572f025bc5))
-* do not redirect to /user on /user namespace ([d95128e](https://github.com/windmill-labs/windmill/commit/d95128e68190fa6f75871f579de906ce82619524))
-* **oauth2:** add google clients ([bc650b0](https://github.com/windmill-labs/windmill/commit/bc650b0ade1d378f815ee01da480a63ddd4501f1))
-* static is undefined by default instead of being empty '' ([fc65162](https://github.com/windmill-labs/windmill/commit/fc651629c7977b5221dbb101f515766b23af9274))
-
-## [1.22.0](https://github.com/windmill-labs/windmill/compare/v1.21.1...v1.22.0) (2022-07-22)
-
-
-### Features
-
-* add delete schedule ([f6d6934](https://github.com/windmill-labs/windmill/commit/f6d69345841f2ec0d06dc32b59840009982c55f2))
-* **backend:** check of no path conflict between flow and flow's primary schedules ([c346339](https://github.com/windmill-labs/windmill/commit/c34633989e41e215d6183e5c887db68d4cc228d3))
-* dynamic template for script inputs in flow ([3c16621](https://github.com/windmill-labs/windmill/commit/3c16621f6b9c2bee1f2630411bd70d075d247974))
-* import and export flow from JSON ([7862ff4](https://github.com/windmill-labs/windmill/commit/7862ff41e25447d7b34aa261187bb98ed3f3105b))
-* more visual cues about trigger scripts ([36606ab](https://github.com/windmill-labs/windmill/commit/36606ab8b675d01b0d38e2dd883b6e42b0987a6c))
-* more visual cues about trigger scripts ([154c2a9](https://github.com/windmill-labs/windmill/commit/154c2a91ca6a4d60b02a44dda5fa23974594018b))
-* rich rendering of flows ([38ffcfe](https://github.com/windmill-labs/windmill/commit/38ffcfeb292c6e9df0c89a4ef5364cdb8e23ccdd))
-
-
-### Bug Fixes
-
-* **deno-client:** make hack for patching openapi-generator more stable ([08ab4d1](https://github.com/windmill-labs/windmill/commit/08ab4d171a286d94e439a89d97115ad2db8e25d9))
-* export json is converted to pull mode ([666e0f6](https://github.com/windmill-labs/windmill/commit/666e0f68d0dd84fce35e6fe1804c90a3c5125057))
-* export json is converted to pull mode + rd fix ([c7528d4](https://github.com/windmill-labs/windmill/commit/c7528d417f276fbdb96751cda547feec7ac6fbc8))
-* **frontend:** filter script by is_trigger and jobs by is_skipped + path fix ([97292d1](https://github.com/windmill-labs/windmill/commit/97292d18fb7158471f1be6ffbd45a612b09a689f))
-* **frontend:** initFlow also reset schemaStore ([5941467](https://github.com/windmill-labs/windmill/commit/5941467ea19938b4d11b56c6f10f529c87cb52a3))
-* **frontend:** remove unecessary step 1 of flows ([f429074](https://github.com/windmill-labs/windmill/commit/f429074528770f5eaebcf1ce687b6431321e169a))
-* improve tooltip ([4be5d37](https://github.com/windmill-labs/windmill/commit/4be5d37a5441555c83eefbea17e86a5df4946749))
-* improve tooltip ([c84b1c9](https://github.com/windmill-labs/windmill/commit/c84b1c9a8c6a03b9689e3405fa87f3c54016914a))
-* placeholder undefined for arginput ([4d01598](https://github.com/windmill-labs/windmill/commit/4d01598e24fca673b0dc83860e151c21ab403b7a))
-
-## [1.21.1](https://github.com/windmill-labs/windmill/compare/v1.21.0...v1.21.1) (2022-07-19)
-
-
-### Bug Fixes
-
-* **deno-client:** make hack for patching openapi-generator more stable ([2f4df43](https://github.com/windmill-labs/windmill/commit/2f4df43a1a798501449e82767d59f08e9cf95146))
-* **python-client:** sed openapi to avoid generator circular dependency ([49f8050](https://github.com/windmill-labs/windmill/commit/49f8050aaf48c15fb79130a06ce754e285d17dd0))
-
-## [1.21.0](https://github.com/windmill-labs/windmill/compare/v1.20.0...v1.21.0) (2022-07-19)
-
-
-### Features
-
-* add run_wait_result to mimic lambda ability ([6ef3754](https://github.com/windmill-labs/windmill/commit/6ef3754759346b8261934a35bd3bf3983872390f))
-
-
-### Bug Fixes
-
-* **backend:** clear env variables before running script ([98a5959](https://github.com/windmill-labs/windmill/commit/98a5959fcca19c54715e78055cf8881496209ac0))
-* consistent exists/{resource} addition + usage in frontend ([ca66d33](https://github.com/windmill-labs/windmill/commit/ca66d33a4297d2f3a105829650a544f4a89c4615))
-* **frontend:** validate username ([9828e54](https://github.com/windmill-labs/windmill/commit/9828e545e9649bc2ac6af598118ef85580fd80f3))
-* list with is_skipped + deno-client fix ([6939f9d](https://github.com/windmill-labs/windmill/commit/6939f9d76b1579f2932e08df3f67dc293c642fd0))
-
-## [1.20.0](https://github.com/windmill-labs/windmill/compare/v1.19.3...v1.20.0) (2022-07-17)
-
-
-### Features
-
-* trigger scripts and have flows being triggered by checking new external events regularly ([#200](https://github.com/windmill-labs/windmill/issues/200)) ([af23b30](https://github.com/windmill-labs/windmill/commit/af23b30c37b4225d6b927644f9612d4861e2d06c))
-
-
-### Bug Fixes
-
-* flow UI back and forth pull/push fix ([8918eb6](https://github.com/windmill-labs/windmill/commit/8918eb6fdb904e23b5dc340db669f6039ed7abb6))
-* flow UI back and forth pull/push fix ([0973859](https://github.com/windmill-labs/windmill/commit/097385981323d5f88a51eb8df0e1114e8cf62727))
-* **frontend:** chrome columns-2 fix for pull/push ([8272b11](https://github.com/windmill-labs/windmill/commit/8272b1110757ee0ed0cee4a7a6de537fcec83de3))
-* **frontend:** createInlineScript only create trigger script if step = 0 ([bd004cf](https://github.com/windmill-labs/windmill/commit/bd004cff0f5150eb043f5446f5697bea43b1508b))
-* HubPicker pick from trigger scripts when relevant ([7e846c3](https://github.com/windmill-labs/windmill/commit/7e846c32a63d9fe2f46f50f7642918cc34459829))
-
-## [1.19.3](https://github.com/windmill-labs/windmill/compare/v1.19.2...v1.19.3) (2022-07-15)
-
-
-### Bug Fixes
-
-* **deno-client:** do not create resource for createInternalPath ([0967c1b](https://github.com/windmill-labs/windmill/commit/0967c1be65a9803e25f7701850be33121eb44d1b))
-
-## [1.19.2](https://github.com/windmill-labs/windmill/compare/v1.19.1...v1.19.2) (2022-07-15)
-
-
-### Bug Fixes
-
-* **deno-client:** handle text/plain parse ([18e33bb](https://github.com/windmill-labs/windmill/commit/18e33bb40739fd699323f2da87de8c9696c0ef6c))
-
-## [1.19.1](https://github.com/windmill-labs/windmill/compare/v1.19.0...v1.19.1) (2022-07-14)
-
-
-### Bug Fixes
-
-* **backend:** create resource would fail if is_oauth was not set ([cd621a6](https://github.com/windmill-labs/windmill/commit/cd621a6285d2aa0e554434998e931e96110464bd))
-* **deno-client:** handle text/plain serialize ([98968ab](https://github.com/windmill-labs/windmill/commit/98968ab039fea89b7525fe7b852ba3d15dee831e))
-
-## [1.19.0](https://github.com/windmill-labs/windmill/compare/v1.18.0...v1.19.0) (2022-07-14)
-
-
-### Features
-
-* add DISABLE_NSJAIL mode ([1943585](https://github.com/windmill-labs/windmill/commit/19435851de0c18fc876a3bd00f3d9153f2719d9b))
-
-
-### Bug Fixes
-
-* add new ca-certificates folders for nsjail ([2eac1ef](https://github.com/windmill-labs/windmill/commit/2eac1ef363b209bb298dcbe7aafb7282ddd2b87a))
-* **frontend:** add arbitrary scopes to connect an app ([372b14e](https://github.com/windmill-labs/windmill/commit/372b14e158bcb10bcfb07d231afeca5cc780661d))
-* write job arguments to file ([#199](https://github.com/windmill-labs/windmill/issues/199)) ([9a6db75](https://github.com/windmill-labs/windmill/commit/9a6db758c15915f5f0027b1d270d621f91b7ae30))
-
-## [1.18.0](https://github.com/windmill-labs/windmill/compare/v1.17.1...v1.18.0) (2022-07-13)
-
-
-### Features
-
-* account part II, handle refresh tokens, clarify oauth UI ([#196](https://github.com/windmill-labs/windmill/issues/196)) ([8403fbb](https://github.com/windmill-labs/windmill/commit/8403fbbc02076bb37dc82b2d26685957b13d036b))
-
-
-### Bug Fixes
-
-* **frontend:** fix path group refresh & create variable path reset ([6a341f5](https://github.com/windmill-labs/windmill/commit/6a341f5dc343df3df6491f8026e87632979faace))
-
-## [1.17.1](https://github.com/windmill-labs/windmill/compare/v1.17.0...v1.17.1) (2022-07-08)
-
-
-### Bug Fixes
-
-* **backend:** set error content-type to text ([cf2dfd7](https://github.com/windmill-labs/windmill/commit/cf2dfd7fe74956d68bdc26dc47557ea6a0ed1ce4))
-* **deno-client:** fix stringify ([5b89abe](https://github.com/windmill-labs/windmill/commit/5b89abe28283238a282da8920580a72f25e5a360))
-* **frontend:** change lsp behavior ([d6e0817](https://github.com/windmill-labs/windmill/commit/d6e0817dc4fe54efd9346698c0ccb39057921d9b))
-* **frontend:** connect an app resource creation ([e400dcc](https://github.com/windmill-labs/windmill/commit/e400dccedd88e3f5e3a9b0ec52fc9883d60c959b))
-* **frontend:** connect an app resource creation ([68c5318](https://github.com/windmill-labs/windmill/commit/68c5318d16c85a01822570c113a4f33c539dc8bf))
-* **frontend:** current hash link ([22eef8a](https://github.com/windmill-labs/windmill/commit/22eef8afab9143bb5b110db8c76e024604106051))
-* **frontend:** fix sendRequest ([5da9819](https://github.com/windmill-labs/windmill/commit/5da9819ca5ce15ef4de9cf4a84affbd581383483))
-* **frontend:** reload editor when language changes for in-flow editor ([72c7890](https://github.com/windmill-labs/windmill/commit/72c7890427736eeeb9a872bf0efd1acc906efd63))
-* **frontend:** sveltekit prerender enabled -> default ([635873a](https://github.com/windmill-labs/windmill/commit/635873a96a586ad8e936526f4f4ebf679519e7fc))
-* in-flow script editor fixes ([466f6b3](https://github.com/windmill-labs/windmill/commit/466f6b339acf70351814c32b8f31d80b8ff1c1b5))
-* in-flow script editor fixes ([5853dfd](https://github.com/windmill-labs/windmill/commit/5853dfd85dca3c80b0edfb58b2866948af8011d5))
-* remove unnecessary v8 snapshot ([d3904fd](https://github.com/windmill-labs/windmill/commit/d3904fd3ebde3a200ccc157a8532dfe1435ae16d))
-
-## [1.17.0](https://github.com/windmill-labs/windmill/compare/v1.16.1...v1.17.0) (2022-07-05)
-
-
-### Features
-
-* in-flow editor mvp ([330b373](https://github.com/windmill-labs/windmill/commit/330b373c24f21b4d9a9b2903e8f1c60ee784ea89))
-
-## [1.16.1](https://github.com/windmill-labs/windmill/compare/v1.16.0...v1.16.1) (2022-07-05)
-
-
-### Bug Fixes
-
-* bump all backend deps by breaking cycling through not using oauth2 ([e4a6378](https://github.com/windmill-labs/windmill/commit/e4a637860133e78cb1675173ccf3ff45e4b08c09))
-* oauth logins used incorrect scope ([1dcba67](https://github.com/windmill-labs/windmill/commit/1dcba67a1f607faabcdfa6f7e94d280c66dd6470))
-* trace errors body ([d092c62](https://github.com/windmill-labs/windmill/commit/d092c622c4efadb1e2799f7dbbe03f825f2b364d))
-
-
-## [1.16.0](https://github.com/windmill-labs/windmill/compare/v1.15.1...v1.16.0) (2022-07-02)
-
-
-### Features
-
-* OAuth "Connect an App" ([#155](https://github.com/windmill-labs/windmill/issues/155)) ([3636866](https://github.com/windmill-labs/windmill/commit/3636866dda8b2e14d61c99a76f0a4e5fa6a37123))
-
-
-### Bug Fixes
-
-* add gitlab to connects ([d4e7c9e](https://github.com/windmill-labs/windmill/commit/d4e7c9e171cd02a7aa0846b43c127720260600b5))
-* diverse frontend fixes
-
-## [1.15.1](https://github.com/windmill-labs/windmill/compare/v1.15.0...v1.15.1) (2022-06-29)
-
-
-### Bug Fixes
-
-* databaseUrlFromResource uses proper database field ([6954580](https://github.com/windmill-labs/windmill/commit/69545808012fa4f5080ec58cf3dff2961a327117))
-
-## [1.15.0](https://github.com/windmill-labs/windmill/compare/v1.14.6...v1.15.0) (2022-06-29)
-
-
-### Features
-
-* Flows Property picker component + Dynamic type inference ([#129](https://github.com/windmill-labs/windmill/issues/129)) ([44b4acf](https://github.com/windmill-labs/windmill/commit/44b4acf4bcfa0c372a9938a9b97d31cceedd9ad9))
-
-## [1.14.6](https://github.com/windmill-labs/windmill/compare/v1.14.5...v1.14.6) (2022-06-27)
-
-
-### Bug Fixes
-
-* add databaseUrlFromResource to deno ([2659e9d](https://github.com/windmill-labs/windmill/commit/2659e9d62b88c2127c969becbc3a61ed2f118069))
-
-## [1.14.5](https://github.com/windmill-labs/windmill/compare/v1.14.4...v1.14.5) (2022-06-27)
-
-
-### Bug Fixes
-
-* index.ts -> mod.ts ([d41913a](https://github.com/windmill-labs/windmill/commit/d41913a440b2034de59437488edc85e38c956d5f))
-* insert getResource proper parenthesis ([e07b5d4](https://github.com/windmill-labs/windmill/commit/e07b5d4f30ea79a99caac4fb63a9ab1f17eaaf74))
-
-## [1.14.4](https://github.com/windmill-labs/windmill/compare/v1.14.3...v1.14.4) (2022-06-27)
-
-
-### Bug Fixes
-
-* windmill deno package index.ts -> mod.ts ([8c0acac](https://github.com/windmill-labs/windmill/commit/8c0acac212d742acee8b7ff0cf6b93cce4187c19))
-
-## [1.14.3](https://github.com/windmill-labs/windmill/compare/v1.14.2...v1.14.3) (2022-06-27)
-
-
-### Bug Fixes
-
-* internal state for script triggers v3 ([31445d7](https://github.com/windmill-labs/windmill/commit/31445d7182a910eab9d699760f2a86ca23d556a4))
-* internal state for script triggers v3 ([22c6347](https://github.com/windmill-labs/windmill/commit/22c6347d8a74d94dc18109390ff5c347a2732823))
-* internal state for script triggers v4 ([63a7401](https://github.com/windmill-labs/windmill/commit/63a7401f248cc37951bbea4dcaedaa6497d6f0b1))
-
-## [1.14.2](https://github.com/windmill-labs/windmill/compare/v1.14.1...v1.14.2) (2022-06-27)
-
-
-### Bug Fixes
-
-* internal state for script triggers v2 ([f9eedc3](https://github.com/windmill-labs/windmill/commit/f9eedc31ed6e5d7e0a8a26633cca9965ac3b6a05))
-
-## [1.14.1](https://github.com/windmill-labs/windmill/compare/v1.14.0...v1.14.1) (2022-06-27)
-
-
-### Bug Fixes
-
-* internal state for script triggers v1 ([6321311](https://github.com/windmill-labs/windmill/commit/6321311112dfa3ef09447f41847b248c0e0dcb46))
-
-## [1.14.0](https://github.com/windmill-labs/windmill/compare/v1.13.0...v1.14.0) (2022-06-27)
-
-
-### Features
-
-* add tesseract bin to worker image ([6de9697](https://github.com/windmill-labs/windmill/commit/6de9697d955a06cfb9c64fdb501b4dfa1bb597ad))
-* deno run with --unstable ([4947661](https://github.com/windmill-labs/windmill/commit/4947661b1d91867c022bb8a10a4be3e91f69352c))
-* internal state for script triggers mvp ([dcdb989](https://github.com/windmill-labs/windmill/commit/dcdb989adb8350974289a0c8d2239b245a6e0d41))
-
-
-### Bug Fixes
-
-* change default per page to 100 ([fdf95a0](https://github.com/windmill-labs/windmill/commit/fdf95a065e83d733ab6a0f02edb4af16c0a1dfb9))
-* deno exit after result logging ([6c622bc](https://github.com/windmill-labs/windmill/commit/6c622bcc32473361e1f7cb1ea7b0b508929bc1b8))
-* improve error handling ([f98f642](https://github.com/windmill-labs/windmill/commit/f98f6429c1e646c0a836f2f73a03a803aa655583))
-* improve error handling ([2efaf21](https://github.com/windmill-labs/windmill/commit/2efaf2191551c1406618c6d60bd37ca6eff84560))
-* schemaPicker does not display editor by default ([fc0c38f](https://github.com/windmill-labs/windmill/commit/fc0c38ffad18a9ceda44cb8406736c14ba4eb4c2))
-* smart assistant reload ([bb946ed](https://github.com/windmill-labs/windmill/commit/bb946ed5519f59adc559d6959c56e61403389c9d))
-
-## [1.13.0](https://github.com/windmill-labs/windmill/compare/v1.12.0...v1.13.0) (2022-06-22)
-
-
-### Features
-
-* better type narrowing for list and array types ([276319d](https://github.com/windmill-labs/windmill/commit/276319d99240dbca5bcc74a1142d99ca823c4da2))
-
-
-### Bug Fixes
-
-* fix webhook path for flows ([906f740](https://github.com/windmill-labs/windmill/commit/906f740a0ddce26743e4669af7a101613131a17c))
-* make email constraint case insensitive ([6dc90a3](https://github.com/windmill-labs/windmill/commit/6dc90a390643fcf6116289596ca1c3149d326797))
-
-## [1.12.0](https://github.com/windmill-labs/windmill/compare/v1.11.0...v1.12.0) (2022-06-14)
-
-
-### Bug Fixes
-
-* more flexible ResourceType MainArgSignature parser ([359ef15](https://github.com/windmill-labs/windmill/commit/359ef15fa2a9024507a71f2c656373925fba3ebe))
-* rename ResourceType -> Resource ([28b5671](https://github.com/windmill-labs/windmill/commit/28b56714023ea69a20f003e08f6c40de64202ac5))
-
-## [1.11.0](https://github.com/windmill-labs/windmill/compare/v1.10.1...v1.11.0) (2022-06-13)
-
-
-### Features
-
-* add DISABLE_NUSER for older kernels ([cce46f9](https://github.com/windmill-labs/windmill/commit/cce46f94404ac5c10407e430fff8cdec3bd7fb2d))
-* add ResourceType<'name'> as deno signature arg type ([f1ee5f3](https://github.com/windmill-labs/windmill/commit/f1ee5f3130cb7b753ccc3ee62169c5e4a8ef7b8b))
-
-
-### Bug Fixes
-
-* force c_ prefix for adding resource type ([9f235c4](https://github.com/windmill-labs/windmill/commit/9f235c404ed62b54a73451b9f9dbddd8f013120d))
-* **frontend:** loadItems not called in script picker ([a59b927](https://github.com/windmill-labs/windmill/commit/a59b92706b24a07cc14288620a9bcdb9402bd134))
-
-## [1.10.1](https://github.com/windmill-labs/windmill/compare/v1.10.0...v1.10.1) (2022-06-12)
-
-
-### Bug Fixes
-
-* python-client verify ssl ([295e28f](https://github.com/windmill-labs/windmill/commit/295e28fd43ef07b739d2c7c85b0ae6819f7d7434))
-
-## [1.10.0](https://github.com/windmill-labs/windmill/compare/v1.9.0...v1.10.0) (2022-06-11)
-
-
-### Features
-
-* alpha hub integration + frontend user store fixes + script client base_url fix ([1a61d50](https://github.com/windmill-labs/windmill/commit/1a61d50076b295fe97e48c2a621dff30802152b1))
-
-## [1.9.0](https://github.com/windmill-labs/windmill/compare/v1.8.6...v1.9.0) (2022-06-05)
-
-
-### Features
-
-* update postgres 13->14 in docker-compose ([479a12f](https://github.com/windmill-labs/windmill/commit/479a12f33ca26bfd1b67bcdd24a64ca26cc6bebe))
-
-
-### Bug Fixes
-
-* remove annoying transitions for scripts and flows ([f2348b5](https://github.com/windmill-labs/windmill/commit/f2348b5526bb8197519685cb57049f74c6f3a11d))
-
-### [1.8.6](https://github.com/windmill-labs/windmill/compare/v1.8.5...v1.8.6) (2022-05-18)
-
-
-### Bug Fixes
-
-* re-release ([d31cd3c](https://github.com/windmill-labs/windmill/commit/d31cd3c52c1b46e821da261f22d0aec872b61fb2))
-
-### [1.8.5](https://github.com/windmill-labs/windmill/compare/v1.8.4...v1.8.5) (2022-05-18)
-
-
-### Bug Fixes
-
-* language field broke flow too ([33fed8e](https://github.com/windmill-labs/windmill/commit/33fed8e04d3abbde371535ecb6e7ba15d103db92))
-
-### [1.8.4](https://github.com/windmill-labs/windmill/compare/v1.8.3...v1.8.4) (2022-05-18)
-
-
-### Bug Fixes
-
-* scripts run was broken due to 1.7 and 1.8 changes. This fix it ([7564d2c](https://github.com/windmill-labs/windmill/commit/7564d2cb1e7f600ede22f333a02a537df381d829))
-
-### [1.8.3](https://github.com/windmill-labs/windmill/compare/v1.8.2...v1.8.3) (2022-05-18)
-
-
-### Bug Fixes
-
-* clean exported deno-client api ([605c2b4](https://github.com/windmill-labs/windmill/commit/605c2b4d11bf072332a38f0c3e24cf6cc9ec7e65))
-
-### [1.8.2](https://github.com/windmill-labs/windmill/compare/v1.8.1...v1.8.2) (2022-05-18)
-
-
-### Bug Fixes
-
-* deno client ([563ba3e](https://github.com/windmill-labs/windmill/commit/563ba3e7f763279a93f619933ac35a1dec3f727a))
-* deno lsp client ([3eed59f](https://github.com/windmill-labs/windmill/commit/3eed59fcb1b172ab13f65c9a0caa0545f5ed91da))
-* deno lsp uses wss instead of ws ([865d728](https://github.com/windmill-labs/windmill/commit/865d728224bed55fe4a2c1905ff2b8c15f4bbe17))
-* starting deno script is now async ([7365a8e](https://github.com/windmill-labs/windmill/commit/7365a8e87bdb1f879eb92125a9e6378a1636637e))
-
-### [1.8.1](https://github.com/windmill-labs/windmill/compare/v1.8.0...v1.8.1) (2022-05-17)
-
-
-### Bug Fixes
-
-* frontend dependencies update ([f793bc4](https://github.com/windmill-labs/windmill/commit/f793bc46d98349a5fea56c7911b6e0720b2b117c))
-
-## [1.8.0](https://github.com/windmill-labs/windmill/compare/v1.7.0...v1.8.0) (2022-05-17)
-
-
-### Features
-
-* Typescript support for scripts (alpha) ([2e1d430](https://github.com/windmill-labs/windmill/commit/2e1d43033f3ad6dbe86338b7a41da7b1120a5ffc))
-
-## [1.7.0](https://github.com/windmill-labs/windmill/compare/v1.6.1...v1.7.0) (2022-05-14)
-
-
-### Features
-
-* self host github oauth ([#46](https://github.com/windmill-labs/windmill/issues/46)) ([5b413d7](https://github.com/windmill-labs/windmill/commit/5b413d7e045d09dc5c5916cb22d82438ec6c92ad))
-
-
-### Bug Fixes
-
-* better error message when saving script ([02c8bea](https://github.com/windmill-labs/windmill/commit/02c8bea0840e492c31ccb8ddd1e5ae9676a534b1))
-
-### [1.6.1](https://github.com/windmill-labs/windmill/compare/v1.6.0...v1.6.1) (2022-05-10)
-
-
-### Bug Fixes
-
-* also store and display "started at" for completed jobs ([#33](https://github.com/windmill-labs/windmill/issues/33)) ([2c28031](https://github.com/windmill-labs/windmill/commit/2c28031e44453740ad8c4b7e3c248173eab34b9c))
-
-## 1.6.0 (2022-05-10)
-
-### Features
-
-* superadmin settings ([7a51f84](https://www.github.com/windmill-labs/windmill/commit/7a51f842f01e17c4d230c060fa0de558553ad3ed))
-* user settings is now at workspace level ([a130806](https://www.github.com/windmill-labs/windmill/commit/a130806e1929267ee40ca443e3dac6e1a5d80da3))
-
-
-### Bug Fixes
-
-* display more than default 30 workspaces as superadmin ([55b5695](https://www.github.com/windmill-labs/windmill/commit/55b5695673912ffe040d3011c020b1002b4e3268))
-
 ## [1.5.0](https://www.github.com/windmill-labs/windmill/v1.5.0) (2022-05-02)

CLA.md

@@ -2,8 +2,8 @@
 
 ## Individual Contributor Non-Exclusive License Agreement
 
-Thank you for your interest in contributing to Windmill Labs, Inc's Windmill
-("We" or "Us").
+Thank you for your interest in contributing to Ruben Fiszel's Windmill ("We" or
+"Us").
 
 The purpose of this contributor agreement ("Agreement") is to clarify and
 document the rights granted by contributors to Us.

Caddyfile

@@ -1,5 +1,4 @@
-{$BASE_URL} {
-        bind {$ADDRESS}
-        reverse_proxy /ws/* http://lsp:3001
-        reverse_proxy /* http://windmill:8000
+{$SITE_URL} {  
+  bind {$ADDRESS}
+  reverse_proxy /* windmill:8000
 }

Dockerfile

@@ -19,7 +19,7 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
 RUN make
 
-FROM node:18-alpine as frontend
+FROM mhart/alpine-node:14 as frontend
 
 # install dependencies
 WORKDIR /frontend
@@ -30,11 +30,8 @@ RUN npm ci
 COPY frontend .
 RUN mkdir /backend
 COPY /backend/openapi.yaml /backend/openapi.yaml
-COPY /openflow.openapi.yaml /openflow.openapi.yaml
 RUN npm run generate-backend-client
-ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
-RUN npm run check
 
 FROM rust:slim-buster as builder
 
@@ -49,7 +46,7 @@ COPY ./backend/.cargo/ .cargo/
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl lld
+    curl
 
 ENV CARGO_INCREMENTAL=1
 
@@ -59,11 +56,11 @@ RUN rm src/*.rs
 RUN rm ./target/release/deps/windmill*
 ENV SQLX_OFFLINE=true
 
-COPY ./backend ./
-COPY ./nsjail /nsjail
+ADD ./backend ./
+ADD ./nsjail /nsjail
 
-COPY --from=frontend /frontend /frontend
-COPY .git/ .git/
+COPY --from=1 /frontend /frontend
+ADD .git/ .git/
 
 RUN cargo build --release
 
@@ -72,11 +69,11 @@ FROM debian:buster-slim
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
+    && apt-get install -y ca-certificates tzdata libpq5 python3 python3-pip \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr \
+    libv8-dev \
     && rm -rf /var/lib/apt/lists/*
 
 ENV TZ=Etc/UTC
@@ -87,16 +84,12 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
     && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
     && make -j 4 && make install
 
-RUN /usr/local/bin/python3 -m pip install pip-tools
-RUN /usr/local/bin/python3 -m pip install nltk
-RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
+RUN python3 -m pip install pip-tools
 
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
-
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -2,7 +2,7 @@
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
 
-Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
+Every file is under copyright (c) Ruben Fiszel 2021 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 

LICENSE-APACHE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2022 Windmill Labs, Inc
+   Copyright 2021 Ruben Fiszel
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -1,4 +1,6 @@
-Copyright (c) 2022  Windmill Labs, Inc
+Ruben Fiszel
+
+Copyright (c) 2021  Ruben Fiszel
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 or the GNU Affero General Public License. Please see

README.md

@@ -2,11 +2,7 @@
   <a href="https://app.windmill.dev"><img src="./imgs/windmill.svg" alt="windmill.dev"></a>
 </p>
 <p align="center">
-    <em>.</em>
-</p>
-<p align=center>
-Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger flows and scripts as internal apps. Convert code to no-code modules and, if the auto-generated UI is not sufficient, use it solely as an highly scalable backend layer. Add automation to your product or build your own no-code tool and delegate the core layer to Windmill
-.
+    <em>Windmill.dev is an OSS developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
 </p>
 <p align="center">
 <a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
@@ -22,42 +18,32 @@ Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks an
 
 ---
 
-**Join the beta (personal workspaces are free forever)**:
+**Join the alpha (personal workspaces are free forever)**:
 <https://app.windmill.dev>
 
 **Documentation**: <https://docs.windmill.dev>
 
 **Discord**: <https://discord.gg/V7PM2YHsPB>
 
-**Hub**: <https://hub.windmill.dev>
-
-**Contributor's guide**: <https://docs.windmill.dev/docs/contributors_guide>
-
-**Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
-
-**[Self-host instruction](#how-to-self-host)**
+**We are hiring**: Software Engineers, DevOps, Solutions Engineers, Growth:
+<https://docs.windmill.dev/hiring>
 
 You can show your support for the project by starring this repo.
 
 ---
 
-Windmill Labs offers commercial licenses and support to convert your existing
-automation and help you scale it in production. If interested, contact
-ruben@windmill.dev (founder of Windmill).
-
----
-
 # Windmill
 
 <p align="center">
-<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
+<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but the API might change,
+especially concerning flows.
 </p>
 
-![Windmill Screenshot](./imgs/windmill.png)
+![Windmill Screenshot](./imgs/windmill.webp)
 
 Windmill is <b>fully open-sourced</b>:
 
-- `community/`, `python-client/` and `deno-client/` are Apache 2.0
+- `community/` and `python-client/` are Apache 2.0
 - backend, frontend and everything else under AGPLv3.
 
 ## What is the general idea behind Windmill
@@ -65,207 +51,77 @@ Windmill is <b>fully open-sourced</b>:
 1. Define a minimal and generic script in Python or Typescript that solve a
    specific task. Here sending an email with SMTP. The code can be defined in
    the provided Web IDE or synchronized with your own github repo:
-   ![Step 1](./imgs/python-script.png)
+   ![Step 1](./imgs/step1.png)
 
 2. Your scripts parameters are automatically parsed and generate a frontend. You
    can narrow down the types during task definition to specify regex for string,
    an enum or a specific format for objects. Each script correspond to an app by
-   itself: ![Step 2](./imgs/arguments.png)
+   itself: ![Step 2](./imgs/step2.png)
 
 3. Make it flow! You can chain your scripts or scripts made by the community
-   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
-   between Windmill and the hub to make it easy to build flows from a soon-to-be
-   exhaustive library of generic modules. In flows, one can pipe output to input
-   using "Dynamic" expressions that are just plain Javascript underneath. Flows
-   can contain for-loops, branching (coming soon). As such and coupled with
-   inputs being able to refer to any step's output, they are actual DAG rather
-   than just linear sequences. They are backed by an open JSON spec we call
-   [OpenFlow](https://docs.windmill.dev/docs/openflow)
-   ![Step 3](./imgs/flow.png)
-
-Both scripts and flows are not restricted to be triggered by the UI. They can be
-triggered by a schedule, watch for changes (using
-[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
-triggered through API with either an async or sync webhook. The latter kind of
-endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
-central place to host, build and run all of your integrations, automation and
-internal apps. We include credentials management and OAuth integration, groups
-and much more!
+   inside flow by piping output to input using "Dynamic" fields that are just
+   plain Javascript. You can also refer to external variables, output from any
+   steps or inputs of the flow itself. The flow parameters then generate
+   automatically an intuitive forms that can be triggered by anyone, like for
+   scripts. ![Step 3](./imgs/step3.png)
 
 ## Layout
 
 - `backend/`: The whole Rust backend
 - `frontend`: The whole Svelte frontend
-- `community/`: Scripts and resource types included in every workspace. It is
-  useful for Python scripts since the [WindmillHub](https://hub.windmill.dev)
-  only allow deno scripts and for sharing resource types that will be included
-  in every workspace.
+- `community/`: Scripts and resource types created and curated by the community,
+  included in every workspace
 - `lsp/`: The lsp asssistant for the monaco editor
 - `nsjail/`: The nsjail configuration files for sandboxing of the scripts'
   execution
 - `python-client/`: The wmill python client used within scripts to interact with
   the windmill platform
-- `deno-client/`: The wmill deno client used within scripts to interact with the
-  windmill platform
 
 ## Stack
 
-- Postgres as the database
-- backend in Rust with the following highly-available and horizontally scalable
+- postgres as the database
+- backend in Rust with the follwing highly-available and horizontally scalable
   architecture:
   - stateless API backend
-  - workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
-    Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
-    interested )
-- frontend in Svelte
-- scripts executions are sandboxed using google's
-  [nsjail](https://github.com/google/nsjail)
-- javascript runtime is the
-  [deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses
-  the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
+  - workers that pull jobs from a queue
+- frontend in svelte
+- scripts executions are sandboxed using google's nsjail
+- javascript runtime is deno_core rust library (which itself uses the rusty_v8
+  and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
 
-## Security
+### Development stack
 
-### Sandboxing and workload isolation
-
-Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
-sandboxing. It is production multi-tenant grade secure. Do not take our word for
-it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
-
-### Secrets, credentials and sensitive values
-
-There is one encryption key per workspace to encrypt the credentials and secrets
-stored in Windmill's K/V store.
-
-In addition, we strongly recommend that you encrypt the whole Postgres database.
-That is what we do at <https://app.windmill.dev>.
-
-## Performance
-
-The performances are great, as long as you do not exceed the parrallelism of the
-workers, we are
-[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
+- caddy is the reverse proxy used for local development, see frontend's
+  Caddyfile and CaddyfileRemote
 
 ## Architecture
 
-<p align="center">
-
-### Big-picture Architecture
-
-<img src="./imgs/diagram.svg">
-
-### Technical Architecture
-
-<img src="./imgs/architecture.svg">
-
-</p>
+![Architecture](./imgs/architecture.svg)
 
 ## How to self-host
 
-We only provide docker-compose setup here. For more advanced setups, like
-compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/how-tos/self_host)
-
-### Docker compose
-
-`docker compose up` with the following docker-compose is sufficient:
-<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
-
-For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
-nsjail will not be able to launch the isolated scripts.
-
-To disable nsjail altogether, set `DISABLE_NSJAIL=true`.
+`docker volume create caddy_data && docker-compose up` with the following
+docker-compose is sufficient:
+<https://github.com/windmill-labs/windmill-server/blob/main/docker-compose.yml>
 
 The default super-admin user is: admin@windmill.dev / changeme
 
 From there, you can create other users (do not forget to change the password!)
 
-### Commercial license
-
-To self-host Windmill, you must respect the terms of the AGPLv3 license which
-you do not need to worry about for personal uses. For business uses, you should
-be fine if you do not re-expose it in any way Windmill to your users and are
-comfortable with AGPLv3.
-
-To re-expose any Windmill parts to your users as a feature of your product, or
-to build a feature on top of Windmill, to comply with AGPLv3 your product must
-be AGPLv3 or you must get a commercial license. Contact us at
-<license@windmill.dev> if you have any doubts.
-
-In addition, a commercial license grants you a dedicated engineer to transition
-your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permissioning features
-such as groups and the ability to create more than one workspace.
-
-### OAuth for self-hosting (very optional)
-
-To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
-the following format:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>"
-  }
-}
-```
-
-and mount it at `/usr/src/app/oauth.json`.
-
-[The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
-
-To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
-welcome contributions!
-
-### Resource types
-
-You will also want to import all the approved resource types from
-[WindmillHub](https://hub.windmill.dev). There is no automatic way to do this
-automatically currently, but it will be possible using a command with the
-upcoming CLI tool.
-
-## Run a local dev setup
-
-### only Frontend
-
-This will use the backend of <https://app.windmill.dev> but your own frontend
-with hot-code reloading.
-
-1. Install [caddy](https://caddyserver.com)
-2. Go to `frontend/`:
-   1. `npm run install`, `npm run generate-backend-client` then `npm run dev`
-   2. In another shell `sudo caddy run --config CaddyfileRemote`
-3. Et voilà, windmill should be available at `http://localhost/`
-
-### Backend + Frontend
-
-See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all running options.
-
-1. Create a Postgres Database for Windmill and create an admin role inside your
-   Postgres setup.
-2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
-   your PATH
-3. Install deno and python3, have the bins at `/usr/bin/deno` and
-   `/usr/local/bin/python3`
-4. Install [caddy](https://caddyserver.com)
-5. Install the [lld linker](https://lld.llvm.org/)
-6. Go to `backend/`:
-   `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
-7. Go to `frontend/`:
-   1. `npm run install`, `npm run generate-backend-client` then `npm run dev`
-   2. In another shell `sudo caddy run --config Caddyfile`
-8. Et voilà, windmill should be available at `http://localhost/`
-
-## Contributors
-
-<a href="https://github.com/windmill-labs/windmill/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=windmill-labs/windmill" />
-</a>
+Detailed instructions for more complex deployments will come soon. For simpler
+docker based ones, the docker-compose.yml file contains all the necessary
+informations.
 
 ## Copyright
 
-Windmill Labs, Inc 2022
+2021 [Ruben Fiszel](https://github.com/rubenfiszel)
+
+### Acknowledgement
+
+This project is inspired from a previous project called
+[Delightool](https://github.com/windmill-labs/delightool-legacy) which was also
+led by [Ruben](https://github.com/rubenfiszel) and with large contribution on
+the frontend from [Malo Marrec](https://github.com/malomarrec) who gave his
+blessing to Windmill.

backend/.cargo/config

@@ -1,3 +1,3 @@
 [build]
-rustflags = ["--cfg", "tokio_unstable", "-C", "link-arg=-fuse-ld=lld"]
+rustflags = ["--cfg", "tokio_unstable"]
 incremental = true

backend/.gitattributes

@@ -1 +0,0 @@
-sqlx-data.json -diff

backend/.gitignore

@@ -1,3 +1,2 @@
 target/
 .env
-oauth.json

backend/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "windmill"
-version = "1.32.0"
-authors = ["Ruben Fiszel <ruben@windmill.dev>"]
+version = "1.5.0"
+authors = ["Ruben Fiszel <ruben@rubenfiszel.com>"]
 edition = "2021"
 
 [build-dependencies]
@@ -17,15 +17,13 @@ tower-http = { version = "^0", features = ["trace"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
-uuid = { version = "^1", features = ["serde", "v4"] }
+uuid = { version = "^0", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"]}
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"]}
 console-subscriber = "^0"
-prometheus = { version = "^0", default-features = false }
-phf = { version = "0.11", features = ["macros"] }
 
 rust-embed = "^6"
 mime_guess = "^2"
@@ -33,33 +31,32 @@ hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
 retainer = "^0"
-rand = "^0"
-rand_core = { version = "^0", features = ["std"] }
+rand = "^0.8.4"
+rand_core = { version = "^0.6.3", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
 rustpython-parser = "^0"
 cron = "^0"
-lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
+external-ip = "^4"
+lettre = { version = "^0.10.0-rc.4", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
 urlencoding = "^2"
+oauth2 = "^4"
 url = "^2"
-async-oauth2 = "^0"
 reqwest = { version = "^0", features = ["json"] }
-time = "^0"
+time = "0.3.7"
 slack-http-verifier = "^0"
 serde_urlencoded = "^0"
 tokio-tar = "^0"
 tempfile = "^3"
-tokio-util = { version = "^0", features = ["io"] }
+tokio-util = { version = "0.7.0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 deno_core = "^0"
+indexmap = "~1.6.2"
 async-recursion = "^1"
-swc_common = "^0"
-swc_ecma_parser = "^0"
-swc_ecma_ast = "^0"
 
 sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
 dotenv = "^0"
-ulid = { version = "^1", features = ["uuid"] }
+ulid = { version = "^0", features = ["uuid"] }
 futures = "^0"

backend/NOTICE

@@ -1,5 +1,5 @@
-Windmill Labs, Inc
+Ruben Fiszel
 
-Copyright (c) 2021  Windmill Labs, Inc
+Copyright (c) 2021  Ruben Fiszel
 
 Source code in this directory is licensed the GNU Affero General Public License. 

backend/build.rs

@@ -0,0 +1,17 @@
+use std::fs::File;
+use std::io::Write;
+
+use deno_core::{JsRuntime, RuntimeOptions};
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    let options = RuntimeOptions {
+        will_snapshot: true,
+        ..Default::default()
+    };
+    let mut runtime = JsRuntime::new(options);
+
+    let mut snap = File::create("v8.snap").expect("can create snap file");
+    snap.write_all(&runtime.snapshot())
+        .expect("can write content to snap");
+}

backend/migrations/20220123221903_first.up.sql

@@ -2,7 +2,6 @@
 create SCHEMA IF NOT exists extensions;
 create extension if not exists "uuid-ossp"      with schema extensions;
 
-
 CREATE TABLE workspace (
     id VARCHAR(50) PRIMARY KEY,
     name VARCHAR(50) NOT NULL,
@@ -206,6 +205,12 @@ CREATE TABLE password (
     company VARCHAR(30)
 );
 
+-- CREATE TABLE invite_code (
+--     code VARCHAR(20) PRIMARY KEY,
+--     seats_left INTEGER NOT NULL DEFAULT 0,
+--     seats_given INTEGER NOT NULL DEFAULT 1
+-- );
+
 
 CREATE TABLE workspace_settings (
     workspace_id VARCHAR(50) PRIMARY KEY REFERENCES workspace(id),
@@ -272,6 +277,17 @@ CREATE TABLE variable (
     CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
 );
 
+-- CREATE TABLE oauth(
+--     id VARCHAR(150) NOT NULL PRIMARY KEY,
+--     owner VARCHAR(50),
+--     workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+--     type VARCHAR(50) NOT NULL,
+--     refresh_token VARCHAR(255), 
+--     access_token VARCHAR(255) NOT NULL
+-- );
+
+-- CREATE INDEX index_oauth ON oauth (workspace_id, type, owner);
+
 CREATE TYPE ACTION_KIND AS ENUM ('create', 'update', 'delete', 'execute');
 
 CREATE TABLE audit (
@@ -404,6 +420,35 @@ CREATE INDEX worker_ping_on_ping_at ON worker_ping (ping_at);
 ALTER TABLE audit ENABLE ROW LEVEL SECURITY;
 CREATE POLICY audit_log_see_own ON audit FOR SELECT
 USING(audit.username = current_setting('session.user') or current_setting('session.is_admin')::boolean);
+-- USING(current_setting('session.is_admin')::boolean);
+
+
+DO
+$do$
+BEGIN
+   IF NOT EXISTS (
+      SELECT FROM pg_catalog.pg_roles
+      WHERE  rolname = 'app') THEN
+
+      CREATE ROLE app LOGIN PASSWORD 'changeme';
+   END IF;
+END
+$do$;
+
+GRANT SELECT ON audit TO app;
+
+REVOKE ALL
+ON ALL TABLES IN SCHEMA public 
+FROM PUBLIC;
+
+GRANT ALL
+ON ALL TABLES IN SCHEMA public 
+TO admin;
+
+ALTER DEFAULT PRIVILEGES 
+    FOR ROLE admin
+    IN SCHEMA public
+    GRANT ALL ON TABLES TO admin;
 
 
 INSERT INTO usr_to_group
@@ -411,10 +456,12 @@ SELECT workspace_id, 'all', username FROM (SELECT workspace_id, username from us
 ;
 
 DROP POLICY audit_log_see_own on audit;
+GRANT ALL ON audit TO app;
 CREATE POLICY see_own ON audit FOR ALL
 USING (audit.username = current_setting('session.user'));
 
 
+GRANT ALL ON queue TO app;
 ALTER TABLE queue ENABLE ROW LEVEL SECURITY;
 
 CREATE POLICY see_own ON queue FOR ALL
@@ -423,6 +470,7 @@ USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(queue.perm
 CREATE POLICY see_member ON queue FOR ALL
 USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(queue.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
 
+GRANT ALL ON completed_job TO app;
 ALTER TABLE completed_job ENABLE ROW LEVEL SECURITY;
 
 
@@ -435,6 +483,16 @@ USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(co
 CREATE POLICY see_member ON completed_job FOR ALL
 USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(completed_job.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
 
+GRANT SELECT ON pipenv to app;
+GRANT SELECT (email, username, is_admin, workspace_id) ON usr to app;
+
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to app;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to admin;
+GRANT SELECT, INSERT ON resource_type to app;
+
+GRANT SELECT ON worker_ping to app;
+GRANT SELECT ON worker_ping to admin;
+
 CREATE POLICY schedule ON audit FOR INSERT
 WITH CHECK (audit.username LIKE 'schedule-%');
 
@@ -450,6 +508,7 @@ $do$
     EXECUTE FORMAT(
       $$
 
+        GRANT ALL ON %1$I TO app;
         ALTER TABLE %1$I ENABLE ROW LEVEL SECURITY;
 
         CREATE POLICY see_starter ON %1$I FOR SELECT
@@ -483,11 +542,13 @@ $do$
   END
 $do$;
 
+GRANT ALL ON group_ TO app;
 ALTER TABLE group_
 ADD COLUMN extra_perms JSONB NOT NULL DEFAULT '{}';
 
 CREATE INDEX group_extra_perms ON group_ USING GIN (extra_perms);
 
+GRANT ALL ON usr_to_group TO app;
 ALTER TABLE usr_to_group ENABLE ROW LEVEL SECURITY;
 
 CREATE POLICY see_extra_perms_user ON usr_to_group FOR ALL
@@ -505,62 +566,10 @@ WITH CHECK (exists(
 DO
 $do$
 BEGIN
-    IF EXISTS (
-        select usesuper from pg_user where usename = CURRENT_USER AND usesuper = 't') 
-    AND NOT EXISTS (
-        SELECT
-        FROM   pg_catalog.pg_roles
-        WHERE  rolname = 'windmill_user') THEN
-
-        LOCK TABLE pg_catalog.pg_roles;
-
-        CREATE ROLE windmill_user;
-
-        GRANT ALL
-        ON ALL TABLES IN SCHEMA public 
-        TO windmill_user;
-
-        GRANT ALL PRIVILEGES 
-        ON ALL SEQUENCES IN SCHEMA public 
-        TO windmill_user;
-
-        ALTER DEFAULT PRIVILEGES 
-            IN SCHEMA public
-            GRANT ALL ON TABLES TO windmill_user;
-
-        ALTER DEFAULT PRIVILEGES 
-            IN SCHEMA public
-            GRANT ALL ON SEQUENCES TO windmill_user;
-
-    END IF;
-END
-$do$;
-
-DO
-$do$
-BEGIN
-    IF EXISTS (select usesuper from pg_user where usename = CURRENT_USER AND usesuper = 't')
-    AND NOT EXISTS (
-        SELECT
-        FROM   pg_catalog.pg_roles
-        WHERE  rolname = 'windmill_admin') THEN
-        CREATE ROLE windmill_admin WITH BYPASSRLS;
-
-        GRANT ALL
-        ON ALL TABLES IN SCHEMA public 
-        TO windmill_admin;
-
-        GRANT ALL PRIVILEGES 
-        ON ALL SEQUENCES IN SCHEMA public 
-        TO windmill_admin;
-
-        ALTER DEFAULT PRIVILEGES 
-            IN SCHEMA public
-            GRANT ALL ON TABLES TO windmill_admin;
-
-        ALTER DEFAULT PRIVILEGES 
-            IN SCHEMA public
-            GRANT ALL ON SEQUENCES TO windmill_admin;
-    END IF;
+   IF NOT EXISTS (
+      SELECT FROM pg_catalog.pg_roles  -- SELECT list can be empty for this
+      WHERE  rolname = 'admin') THEN
+      CREATE ROLE admin WITH BYPASSRLS LOGIN PASSWORD 'changeme';
+   END IF;
 END
 $do$;

backend/migrations/20220428085013_private_key.up.sql

@@ -9,5 +9,8 @@ CREATE TABLE workspace_key (
     PRIMARY KEY (workspace_id, kind)
 );
 
+GRANT SELECT ON workspace_key TO app;
+GRANT SELECT ON workspace_key TO admin;
+
 INSERT INTO workspace_key SELECT id as workspace_id, 'cloud' as kind, 'changeme' as key FROM workspace;
 

backend/migrations/20220504193929_typescript_support.down.sql

@@ -1,12 +0,0 @@
--- Add down migration script here
-DROP TYPE SCRIPT_LANG;
-
-ALTER TABLE script
-DROP COLUMN language SCRIPT_LANG;
-
-ALTER TABLE queue
-DROP COLUMN language SCRIPT_LANG;
-
-ALTER TABLE completed_job
-DROP COLUMN language SCRIPT_LANG;
-

backend/migrations/20220504193929_typescript_support.up.sql

@@ -1,11 +0,0 @@
--- Add up migration script here
-CREATE TYPE SCRIPT_LANG AS ENUM ('python3', 'deno');
-
-ALTER TABLE script
-ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
-
-ALTER TABLE queue
-ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
-
-ALTER TABLE completed_job
-ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';

backend/migrations/20220508152326_create_role_if_not_exists.up.sql

@@ -1 +1,24 @@
 -- Add up migration script here
+DO
+$do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'app') THEN
+        CREATE ROLE app LOGIN PASSWORD 'changeme';
+    END IF;
+END
+$do$;
+
+DO
+$do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'admin') THEN
+        CREATE ROLE admin LOGIN PASSWORD 'changeme';
+    END IF;
+END
+$do$;

backend/migrations/20220509191845_started_at_completed_job.down.sql

@@ -1,3 +0,0 @@
--- Add down migration script here
-ALTER TABLE completed_job
-DROP COLUMN started_at;

backend/migrations/20220509191845_started_at_completed_job.up.sql

@@ -1,4 +0,0 @@
--- Add up migration script here
-ALTER TABLE completed_job
-ADD COLUMN started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW();
-

backend/migrations/20220518211432_make_lang_nullable.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220518211432_make_lang_nullable.up.sql

@@ -1,6 +0,0 @@
--- Add up migration script here
-ALTER TABLE queue
-ALTER COLUMN language DROP NOT NULL;
-
-ALTER TABLE completed_job
-ALTER COLUMN language DROP NOT NULL;

backend/migrations/20220610181005_add_script_hub.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220610181005_add_script_hub.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TYPE JOB_KIND ADD VALUE 'script_hub';

backend/migrations/20220615151034_accounts.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220615151034_accounts.up.sql

@@ -1,13 +0,0 @@
--- Add up migration script here
-
-CREATE TABLE account (
-    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-    id SERIAL NOT NULL,
-    expires_at TIMESTAMP,
-    refresh_token VARCHAR(255),
-    PRIMARY KEY (workspace_id, id)
-);
-
-ALTER TABLE resource ADD COLUMN account INTEGER;
-ALTER TABLE variable ADD COLUMN account INTEGER;
-ALTER TABLE password ALTER COLUMN login_type TYPE VARCHAR(50);

backend/migrations/20220620210708_regex_fix.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220620210708_regex_fix.up.sql

@@ -1,9 +0,0 @@
--- Add up migration script here
-
-ALTER TABLE usr DROP CONSTRAINT proper_email;
-ALTER TABLE usr ADD CONSTRAINT proper_email
-CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');
-
-ALTER TABLE workspace_invite DROP CONSTRAINT proper_email;
-ALTER TABLE workspace_invite ADD CONSTRAINT proper_email
-CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');

backend/migrations/20220624170622_workspace_premium.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220624170622_workspace_premium.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE workspace ADD COLUMN premium BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20220624180013_workspace_premium_grant.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220624180013_workspace_premium_grant.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-

backend/migrations/20220707094935_add_client_to_account.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220707094935_add_client_to_account.up.sql

@@ -1,20 +0,0 @@
--- Add up migration script here
-ALTER TABLE account ADD COLUMN owner VARCHAR(50) NOT NULL;
-ALTER TABLE account ADD COLUMN client VARCHAR(50) NOT NULL;
-ALTER TABLE resource ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
-ALTER TABLE variable ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
-ALTER TABLE resource DROP COLUMN account;
-
-ALTER TABLE account ALTER COLUMN expires_at TYPE TIMESTAMP WITH TIME ZONE;
-
-ALTER TABLE account ALTER COLUMN expires_at SET NOT NULL;
-ALTER TABLE account ALTER COLUMN refresh_token SET NOT NULL;
-
-ALTER TABLE account ENABLE ROW LEVEL SECURITY;
-
-
-CREATE POLICY see_own ON account FOR ALL
-USING (SPLIT_PART(account.owner, '/', 1) = 'u' AND SPLIT_PART(account.owner, '/', 2) = current_setting('session.user'));
-
-CREATE POLICY see_member ON account FOR ALL
-USING (SPLIT_PART(account.owner, '/', 1) = 'g' AND SPLIT_PART(account.owner, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20220713142758_script_trigger.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220713142758_script_trigger.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE script ADD COLUMN trigger_reco_interval INTEGER;
-ALTER TABLE completed_job ADD COLUMN is_skipped BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20220719101639_script_trigger_2.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220719101639_script_trigger_2.up.sql

@@ -1,4 +0,0 @@
--- Add up migration script here
-ALTER TABLE script DROP COLUMN trigger_reco_interval;
-ALTER TABLE script ADD COLUMN is_trigger BOOLEAN NOT NULL DEFAULT false;
-

backend/migrations/20220728222351_job_duration_ms.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220728222351_job_duration_ms.up.sql

@@ -1,4 +0,0 @@
-ALTER TABLE completed_job
-     RENAME duration to duration_ms;
-UPDATE completed_job
-   SET duration_ms = duration_ms * 1000;

backend/migrations/20220816185849_remove_non_admin_users.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220816185849_remove_non_admin_users.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-DELETE FROM password WHERE email = 'user@windmill.dev' OR email = 'ruben@windmill.dev';

backend/migrations/20220818125523_longername.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20220818125523_longername.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE queue ALTER COLUMN created_by TYPE varchar(255);
-ALTER TABLE completed_job ALTER COLUMN created_by TYPE varchar(255);

backend/oauth_connect.json

@@ -1,94 +0,0 @@
-{
-    "github": {
-        "auth_url": "https://github.com/login/oauth/authorize",
-        "token_url": "https://github.com/login/oauth/access_token",
-        "scopes": [
-            "workflow",
-            "repo"
-        ]
-    },
-    "gitlab": {
-        "auth_url": "https://gitlab.com/oauth/authorize",
-        "token_url": "https://gitlab.com/oauth/token",
-        "scopes": [
-            "api"
-        ]
-    },
-    "bitbucket": {
-        "auth_url": "https://bitbucket.org/site/oauth2/authorize",
-        "token_url": "https://bitbucket.org/site/oauth2/access_token",
-        "scopes": [
-            "repository"
-        ]
-    },
-    "slack": {
-        "auth_url": "https://slack.com/oauth/authorize",
-        "token_url": "https://slack.com/api/oauth.access",
-        "scopes": [
-            "chat:write:user"
-        ]
-    },
-    "gsheets": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/spreadsheets"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gdrive": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/drive"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gmail": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/gmail.send"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcal": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/calendar.events"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcloud": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/cloud-platform"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "basecamp": {
-        "auth_url": "https://launchpad.37signals.com/authorization/new",
-        "token_url": "https://launchpad.37signals.com/authorization/token",
-        "scopes": [],
-        "extra_params": {
-            "type": "web_server"
-        }
-    }
-}

backend/oauth_login.json

@@ -1,23 +0,0 @@
-{
-    "github": {
-        "auth_url": "https://github.com/login/oauth/authorize",
-        "token_url": "https://github.com/login/oauth/access_token",
-        "scopes": [
-            "user:email"
-        ]
-    },
-    "gitlab": {
-        "auth_url": "https://gitlab.com/oauth/authorize",
-        "token_url": "https://gitlab.com/oauth/token",
-        "scopes": [
-            "read_user"
-        ]
-    },
-    "google": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/userinfo.email"
-        ]
-    }
-}

backend/overwrite_migrations.py

@@ -1,24 +0,0 @@
-import sys
-import os
-import subprocess
-
-database_url = sys.argv[1]
-
-print(f"database_url: {database_url}")
-
-if database_url is None:
-    print("Please provide a database url")
-    sys.exit(1)
-
-for f in os.listdir("migrations"):
-    if f.endswith(".up.sql"):
-        version = f.split("_")[0]
-        cmd = f"cat migrations/{f} | openssl dgst -sha384 | cut -d ' ' -f 2"
-        ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
-        digest = ps.communicate()[0].decode("utf-8").strip()
-
-        cmd = f"psql '{database_url}' -c \"UPDATE _sqlx_migrations SET checksum = '\\x{digest}' WHERE version = {version};\""
-        ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
-        out = ps.communicate()[0].decode("utf-8").strip()
-        print(version, digest, out)
-

backend/rustfmt.toml

@@ -1,5 +1,7 @@
+imports_granularity = "Crate"
 max_width = 100
 use_small_heuristics = "Default"
-match_arm_leading_pipes="Preserve"
-struct_lit_width=100
-struct_variant_width=100
+indent_style = "Block"
+fn_single_line = false
+force_multiline_blocks = true
+format_strings = true

backend/src/audit.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -65,12 +64,13 @@ pub async fn audit_log<'c>(
     let p_json: serde_json::Value = serde_json::to_value(&parameters).unwrap();
 
     tracing::info!(
+        username = username,
+        kind = "audit",
         operation = operation,
+        workspace = w_id,
         action_kind = ?action_kind,
         resource = resource,
-        parameters = %p_json,
-        workspace_id = w_id,
-        username = username,
+        parameters = %p_json
     );
     sqlx::query(
         "INSERT INTO audit

backend/src/client.rs

@@ -31,17 +31,14 @@ pub async fn get_resource(
     base_url: &str,
 ) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let client = reqwest::Client::new();
-    let res = client
+    let result = client
         .get(format!(
             "{base_url}/api/w/{workspace}/resources/get_value/{path}"
         ))
         .bearer_auth(token)
         .send()
+        .await?
+        .json::<Option<serde_json::Value>>()
         .await?;
-    if res.status().is_success() {
-        let value = res.json::<Option<serde_json::Value>>().await?;
-        Ok(value)
-    } else {
-        Err(Error::NotFound(format!("Resource not found at {path}")))?
-    }
+    Ok(result)
 }

backend/src/db.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -12,9 +11,9 @@ use std::time::Duration;
 
 pub type DB = Pool<Postgres>;
 
-pub async fn connect(database_url: &str, max_connections: u32) -> Result<DB, Error> {
+pub async fn connect(database_url: &str) -> Result<DB, Error> {
     PgPoolOptions::new()
-        .max_connections(max_connections)
+        .max_connections(100)
         .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
         .connect(database_url)
         .await
@@ -30,6 +29,19 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
     Ok(())
 }
 
+pub async fn setup_app_user(db: &DB, password: &str) -> Result<(), Error> {
+    let mut tx = db.begin().await?;
+
+    sqlx::query(&format!("ALTER USER app WITH PASSWORD '{}'", password))
+        .execute(&mut tx)
+        .await?;
+    sqlx::query(&format!("ALTER USER admin WITH PASSWORD '{}'", password))
+        .execute(&mut tx)
+        .await?;
+    tx.commit().await?;
+
+    Ok(())
+}
 #[derive(Clone)]
 pub struct UserDB {
     db: DB,
@@ -45,13 +57,9 @@ impl UserDB {
         authed: &Authed,
     ) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
         let mut tx = self.db.begin().await?;
-        let user = if authed.is_admin {
-            "windmill_admin"
-        } else {
-            "windmill_user"
-        };
+        let user = if authed.is_admin { "admin" } else { "app" };
 
-        sqlx::query(&format!("SET LOCAL ROLE {}", user))
+        sqlx::query(&format!("SET LOCAL SESSION AUTHORIZATION {}", user))
             .execute(&mut tx)
             .await?;
 

backend/src/error.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -45,7 +44,7 @@ pub enum Error {
     HexErr(#[from] hex::FromHexError),
     #[error("Migrating database: {0}")]
     DatabaseMigration(#[from] MigrateError),
-    #[error(transparent)]
+    #[error("{0}")]
     Anyhow(#[from] anyhow::Error),
 }
 
@@ -63,11 +62,6 @@ impl IntoResponse for Error {
             Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST,
             _ => StatusCode::INTERNAL_SERVER_ERROR,
         };
-        tracing::error!(error = e.to_string());
-        Response::builder()
-            .header("Content-Type", "text/plain")
-            .status(status)
-            .body(body)
-            .unwrap()
+        Response::builder().status(status).body(body).unwrap()
     }
 }

backend/src/external_ip.rs

@@ -1,14 +0,0 @@
-//! Used to determine the internet address that connections from workers will appear to come from.
-//!
-//! For users writing scripts to access their infrastructure with firewalls requiring incoming
-//! connections to be from whitelisted IP addresses.
-
-use reqwest::Result;
-
-pub async fn get_ip() -> Result<String> {
-    reqwest::get("https://hub.windmill.dev/getip")
-        .await?
-        .error_for_status()?
-        .text()
-        .await
-}

backend/src/fixtures/base.sql

@@ -1,19 +0,0 @@
--- used for backend automated testing
--- https://docs.rs/sqlx/latest/sqlx/attr.test.html
-
-INSERT INTO workspace
-            (id,               name,             owner,       domain)
-     VALUES ('test-workspace', 'test-workspace', 'test-user', null);
-
-CREATE FUNCTION "notify_insert_on_completed_job" ()
-RETURNS TRIGGER AS $$
-BEGIN
-    PERFORM pg_notify('insert on completed_job', NEW.id::text);
-    RETURN new;
-END;
-$$ LANGUAGE PLPGSQL;
-
-  CREATE TRIGGER "notify_insert_on_completed_job"
- AFTER INSERT ON "completed_job"
-    FOR EACH ROW
-EXECUTE FUNCTION "notify_insert_on_completed_job" ();

backend/src/flow.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -8,26 +7,24 @@
 
 use std::collections::HashMap;
 
-use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
-    extract::{Extension, Host, Path, Query},
+    extract::{Extension, Path, Query},
     routing::{get, post},
     Json, Router,
 };
 use serde::{Deserialize, Serialize};
 use sql_builder::SqlBuilder;
-use sqlx::{FromRow, Postgres, Transaction};
+use sqlx::FromRow;
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::{UserDB, DB},
-    error::{self, to_anyhow, Error, JsonResult, Result},
-    jobs::RawCode,
+    db::UserDB,
+    error::{Error, JsonResult, Result},
     scripts::Schema,
     users::Authed,
-    utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath},
+    utils::{Pagination, StripPath},
 };
 
 pub fn workspaced_service() -> Router {
@@ -37,13 +34,6 @@ pub fn workspaced_service() -> Router {
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
-        .route("/exists/*path", get(exists_flow_by_path))
-}
-
-pub fn global_service() -> Router {
-    Router::new()
-        .route("/hub/list", get(list_hub_flows))
-        .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
 #[derive(FromRow, Serialize)]
@@ -69,24 +59,19 @@ pub struct NewFlow {
     pub schema: Option<Schema>,
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone)]
+#[derive(Deserialize, Serialize)]
 pub struct FlowValue {
     pub modules: Vec<FlowModule>,
     pub failure_module: Option<FlowModule>,
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone)]
+#[derive(Deserialize, Serialize)]
 pub struct FlowModule {
-    #[serde(default)]
-    #[serde(alias = "input_transform")]
-    pub input_transforms: HashMap<String, InputTransform>,
+    pub input_transform: HashMap<String, InputTransform>,
     pub value: FlowModuleValue,
-    pub stop_after_if_expr: Option<String>,
-    pub skip_if_stopped: Option<bool>,
-    pub summary: Option<String>,
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+#[derive(Deserialize, Serialize)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
@@ -94,31 +79,17 @@ pub struct FlowModule {
 pub enum InputTransform {
     Static { value: serde_json::Value },
     Javascript { expr: String },
+    Resource { path: String },
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
 )]
 pub enum FlowModuleValue {
-    Script {
-        path: String,
-    },
-    ForloopFlow {
-        iterator: InputTransform,
-        value: Box<FlowValue>,
-        #[serde(default = "default_true")]
-        skip_failures: bool,
-    },
-    Flow {
-        path: String,
-    },
-    RawScript(RawCode),
-}
-
-fn default_true() -> bool {
-    true
+    Script { path: String },
+    Flow { path: String },
 }
 
 #[derive(Deserialize)]
@@ -150,7 +121,7 @@ async fn list_flows(
             "edited_by",
             "edited_at",
             "archived",
-            "null schema",
+            "schema",
             "extra_perms",
         ])
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
@@ -179,43 +150,6 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(
-    Authed { email, username, .. }: Authed,
-    Extension(http_client): Extension<Client>,
-    Host(host): Host,
-) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
-        http_client,
-        "https://hub.windmill.dev/searchFlowData?approved=true",
-        email,
-        username,
-        host,
-    )
-    .await?;
-    Ok(Json(flows))
-}
-
-pub async fn get_hub_flow_by_id(
-    Authed { email, username, .. }: Authed,
-    Path(id): Path<i32>,
-    Extension(http_client): Extension<Client>,
-    Host(host): Host,
-) -> JsonResult<serde_json::Value> {
-    let value = http_get_from_hub(
-        http_client,
-        &format!("https://hub.windmill.dev/flows/{id}/json"),
-        email,
-        username,
-        host,
-        false,
-    )
-    .await?
-    .json()
-    .await
-    .map_err(to_anyhow)?;
-    Ok(Json(value))
-}
-
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -225,17 +159,15 @@ async fn create_flow(
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
-
     sqlx::query!(
-        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json)",
+        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::json)",
         w_id,
         nf.path,
         nf.summary,
         nf.description,
         nf.value,
         &authed.username,
+        &chrono::Utc::now(),
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
     )
     .execute(&mut tx)
@@ -261,29 +193,6 @@ async fn create_flow(
     Ok(nf.path.to_string())
 }
 
-async fn check_schedule_conflict<'c>(
-    tx: &mut Transaction<'c, Postgres>,
-    w_id: &str,
-    path: &str,
-) -> error::Result<()> {
-    let exists_flow = sqlx::query_scalar!(
-        "SELECT EXISTS (SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2 AND path != \
-         script_path)",
-        path,
-        w_id
-    )
-    .fetch_one(tx)
-    .await?
-    .unwrap_or(false);
-    if exists_flow {
-        return Err(error::Error::BadConfig(format!(
-            "A flow cannot have the same path as a schedule if the schedule does not trigger that \
-             same flow: {path}",
-        )));
-    };
-    Ok(())
-}
-
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -293,17 +202,15 @@ async fn update_flow(
     let mut tx = user_db.begin(&authed).await?;
 
     let flow_path = flow_path.to_path();
-    check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
-
     let schema = nf.schema.map(|x| x.0);
     let flow = sqlx::query_scalar!(
-        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6 WHERE path = $7 AND workspace_id = $8 RETURNING path",
+        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = $6, schema = $7 WHERE path = $8 AND workspace_id = $9 RETURNING path",
         nf.path,
         nf.summary,
         nf.description,
         nf.value,
         &authed.username,
+        &chrono::Utc::now(),
         schema,
         flow_path,
         w_id,
@@ -353,25 +260,6 @@ async fn get_flow_by_path(
     Ok(Json(flow))
 }
 
-async fn exists_flow_by_path(
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<bool> {
-    let path = path.to_path();
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id \
-         = 'starter'))",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -414,78 +302,25 @@ mod tests {
         let mut hm = HashMap::new();
         hm.insert(
             "test".to_owned(),
-            InputTransform::Static { value: serde_json::json!("test2") },
+            InputTransform::Static {
+                value: serde_json::json!("test2"),
+            },
         );
         let fv = FlowValue {
-            modules: vec![
-                FlowModule {
-                    input_transforms: hm,
-                    value: FlowModuleValue::Script { path: "test".to_string() },
-                    stop_after_if_expr: None,
-                    skip_if_stopped: Some(false),
-                    summary: None,
+            modules: vec![FlowModule {
+                input_transform: hm,
+                value: FlowModuleValue::Script {
+                    path: "test".to_string(),
                 },
-                FlowModule {
-                    input_transforms: HashMap::new(),
-                    value: FlowModuleValue::RawScript(RawCode {
-                        content: "test".to_string(),
-                        language: crate::scripts::ScriptLang::Deno,
-                        path: None,
-                    }),
-                    stop_after_if_expr: Some("foo = 'bar'".to_string()),
-                    skip_if_stopped: None,
-                    summary: None,
-                },
-                FlowModule {
-                    input_transforms: [(
-                        "iterand".to_string(),
-                        InputTransform::Static { value: serde_json::json!(vec![1, 2, 3]) },
-                    )]
-                    .into(),
-                    value: FlowModuleValue::ForloopFlow {
-                        iterator: InputTransform::Static { value: serde_json::json!([1, 2, 3]) },
-                        value: Box::new(FlowValue { modules: vec![], failure_module: None }),
-                        skip_failures: true,
-                    },
-                    stop_after_if_expr: Some("previous.isEmpty()".to_string()),
-                    skip_if_stopped: None,
-                    summary: None,
-                },
-            ],
+            }],
             failure_module: Some(FlowModule {
-                input_transforms: HashMap::new(),
-                value: FlowModuleValue::Flow { path: "test".to_string() },
-                stop_after_if_expr: Some("previous.isEmpty()".to_string()),
-                skip_if_stopped: None,
-                summary: None,
+                input_transform: HashMap::new(),
+                value: FlowModuleValue::Flow {
+                    path: "test".to_string(),
+                },
             }),
         };
         println!("{}", serde_json::json!(fv).to_string());
         Ok(())
     }
-
-    #[test]
-    fn test_back_compat() {
-        /* renamed input_transform -> input_transforms but should deserialize old name */
-        let s = r#"
-        {
-            "value": {
-                "type": "rawscript",
-                "content": "def main(n): return",
-                "language": "python3"
-            },
-            "input_transform": {
-                "n": {
-                    "expr": "flow_input.iter.value",
-                    "type": "javascript"
-                }
-            }
-        }
-        "#;
-        let module: FlowModule = serde_json::from_str(s).unwrap();
-        assert_eq!(
-            module.input_transforms["n"],
-            InputTransform::Javascript { expr: "flow_input.iter.value".to_string() }
-        );
-    }
 }

backend/src/granular_acls.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -47,8 +46,7 @@ async fn add_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
-         true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(write.unwrap_or(false))
     .bind(path)
@@ -76,8 +74,7 @@ async fn remove_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
-         workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(owner)
     .bind(path)

backend/src/groups.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -259,7 +258,9 @@ async fn add_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username { username: user_username }): Json<Username>,
+    Json(Username {
+        username: user_username,
+    }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
@@ -293,7 +294,9 @@ async fn remove_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username { username: user_username }): Json<Username>,
+    Json(Username {
+        username: user_username,
+    }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 

backend/src/js_eval.rs

@@ -1,60 +1,57 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use deno_core::serde_v8;
+use deno_core::v8;
+use deno_core::v8::IsolateHandle;
+use deno_core::JsRuntime;
+use deno_core::OpState;
+use deno_core::RuntimeOptions;
+use deno_core::Snapshot;
+use deno_core::ZeroCopyBuf;
 use itertools::Itertools;
 use regex::Regex;
 use serde_json::Value;
-use tokio::{sync::oneshot, time::timeout};
+use tokio::sync::oneshot;
+use tokio::time::timeout;
 
-use crate::{client, error::Error};
-
-pub struct EvalCreds {
-    pub workspace: String,
-    pub token: String,
-}
+use crate::client;
+use crate::error::Error;
 
 pub async fn eval_timeout(
     expr: String,
     env: Vec<(String, serde_json::Value)>,
-    creds: Option<EvalCreds>,
+    workspace: &str,
+    token: &str,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr2 = expr.clone();
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
+    let (workspace, token) = (workspace.to_string().clone(), token.to_string().clone());
     timeout(
         std::time::Duration::from_millis(2000),
         tokio::task::spawn_blocking(move || {
-            let mut ops = vec![];
+            let buffer = include_bytes!("../v8.snap");
 
-            if creds.is_some() {
-                ops.extend([
-                    // An op for summing an array of numbers
-                    // The op-layer automatically deserializes inputs
-                    // and serializes the returned Result & value
-                    op_variable::decl(),
-                    op_resource::decl(),
-                ])
-            }
-
-            if !steps.is_empty() {
-                ops.push(op_get_result::decl())
-            }
-
-            let ext = Extension::builder().ops(ops).build();
             // Use our snapshot to provision our new runtime
             let options = RuntimeOptions {
-                extensions: vec![ext],
-                //                startup_snapshot: Some(Snapshot::Static(buffer)),
+                startup_snapshot: Some(Snapshot::Static(buffer)),
                 ..Default::default()
             };
-
             let mut js_runtime = JsRuntime::new(options);
+            js_runtime.register_op("variable", deno_core::op_async(op_variable));
+            js_runtime.register_op("resource", deno_core::op_async(op_resource));
+            if !steps.is_empty() {
+                js_runtime.register_op("result", deno_core::op_async(op_get_result));
+            }
+            js_runtime.sync_ops_cache();
 
             sender
                 .send(js_runtime.v8_isolate().thread_safe_handle())
@@ -71,7 +68,8 @@ pub async fn eval_timeout(
                 .into_iter()
                 .fold(expr, replace_with_await);
 
-            let r = runtime.block_on(eval(&mut js_runtime, &expr, env, creds, steps))?;
+            let r =
+                runtime.block_on(eval(&mut js_runtime, &expr, env, &workspace, &token, steps))?;
 
             Ok(r) as anyhow::Result<Value>
         }),
@@ -121,7 +119,8 @@ async fn eval(
     context: &mut JsRuntime,
     expr: &str,
     env: Vec<(String, serde_json::Value)>,
-    creds: Option<EvalCreds>,
+    workspace: &str,
+    token: &str,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr = expr.trim();
@@ -132,12 +131,14 @@ async fn eval(
             .join("\n"),
         expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
     );
-    let (steps_code, api_code) = if let Some(EvalCreds { workspace, token }) = creds {
-        let steps_code = if !steps.is_empty() {
-            format!(
-                r#"
+    let steps_code = if !steps.is_empty() {
+        format!(
+            r#"
 let steps = [{}];
 async function step(n) {{
+    if (n == 0) {{
+        return flow_input;
+    }}
     if (n == -1) {{
         return previous_result;
     }}
@@ -147,55 +148,42 @@ async function step(n) {{
         n = n % steps.length + steps.length;
     }}
     let id = steps[n];
-    return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
+    return await Deno.core.opAsync("result", [workspace, id, token, base_url]);
 }}"#,
-                steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
-            )
-        } else {
-            String::new()
-        };
-
-        let api_code = format!(
-            r#"
-let workspace = "{workspace}";
-let base_url = "{}";
-async function variable(path) {{
-    let token = "{token}";
-    return await Deno.core.opAsync("op_variable", [workspace, path, token, base_url]);
-}}
-async function resource(path) {{
-    let token = "{token}";
-    return await Deno.core.opAsync("op_resource", [workspace, path, token, base_url]);
-}}
-        "#,
-            std::env::var("BASE_INTERNAL_URL")
-                .unwrap_or_else(|_| "http://missing-base-url".to_string()),
-        );
-        (steps_code, api_code)
+            steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
+        )
     } else {
-        (String::new(), String::new())
+        "".to_string()
     };
 
     let code = format!(
         r#"
-{api_code}
+let workspace = "{workspace}";
+let base_url = "{}";
+async function variable(path) {{
+    let token = "{token}";
+    return await Deno.core.opAsync("variable", [workspace, path, token, base_url]);
+}}
+async function resource(path) {{
+    let token = "{token}";
+    return await Deno.core.opAsync("resource", [workspace, path, token, base_url]);
+}}
 {}
 {steps_code}
 (async () => {{ 
     {expr} 
 }})()
         "#,
+        std::env::var("BASE_INTERNAL_URL")
+            .unwrap_or_else(|_| "http://missing-base-url".to_string()),
         env.into_iter()
-            .map(|(a, b)| {
-                format!(
-                    "let {a} = {};\n",
-                    serde_json::to_string(&b)
-                        .unwrap_or_else(|_| "\"error serializing value\"".to_string())
-                )
-            })
+            .map(|(a, b)| format!(
+                "let {a} = {};\n",
+                serde_json::to_string(&b)
+                    .unwrap_or_else(|_| "\"error serializing value\"".to_string())
+            ))
             .join(""),
     );
-    tracing::debug!("{}", code);
     let global = context.execute_script("<anon>", &code)?;
     let global = context.resolve_value(global).await?;
 
@@ -216,8 +204,11 @@ async function resource(path) {{
 //     Ok(path)
 // }
 
-#[op]
-async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
+async fn op_variable(
+    _state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+    _buf: Option<ZeroCopyBuf>,
+) -> Result<String, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -225,8 +216,11 @@ async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
     client::get_variable(workspace, path, token, &base_url).await
 }
 
-#[op]
-async fn op_get_result(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
+async fn op_get_result(
+    _state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+    _buf: Option<ZeroCopyBuf>,
+) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let id = &args[1];
     let token = &args[2];
@@ -244,8 +238,11 @@ async fn op_get_result(args: Vec<String>) -> Result<Option<serde_json::Value>, a
     Ok(result)
 }
 
-#[op]
-async fn op_resource(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
+async fn op_resource(
+    _state: Rc<RefCell<OpState>>,
+    args: Vec<String>,
+    _buf: Option<ZeroCopyBuf>,
+) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -270,7 +267,7 @@ mod tests {
         let code = "value.test + params.test";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, vec![]).await?;
+        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
         assert_eq!(res, json!(4));
         Ok(())
     }
@@ -283,7 +280,7 @@ mod tests {
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, None, vec![]).await?;
+        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
         assert_eq!(res, json!("my 5\nmultiline template"));
         Ok(())
     }
@@ -294,10 +291,10 @@ multiline template`";
             ("params".to_string(), json!({"test": 2})),
             ("value".to_string(), json!({"test": 2})),
         ];
-        let code = r#"params.test"#;
+        let code = r#"variable("test")"#;
 
-        let res = eval_timeout(code.to_string(), env, None, vec![]).await?;
-        assert_eq!(res, json!(2));
+        let res = eval_timeout(code.to_string(), env, "workspace", "token", vec![]).await?;
+        assert_eq!(res, json!("test"));
         Ok(())
     }
 }

backend/src/lib.rs

@@ -1,24 +1,24 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use anyhow::Context;
+use ::oauth2::basic::BasicClient;
 use argon2::Argon2;
-use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router};
+use axum::{extract::extractor_middleware, handler::Handler, routing::get, Extension, Router};
 use db::DB;
-use futures::FutureExt;
 use git_version::git_version;
-use rand::Rng;
+use hyper::Response;
 use slack_http_verifier::SlackVerifier;
-use std::{net::SocketAddr, sync::Arc};
+use std::{collections::HashMap, net::SocketAddr, sync::Arc};
+use tokio::sync::Mutex;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
-use tower_http::trace::TraceLayer;
-
+use tower_http::trace::{MakeSpan, OnResponse, TraceLayer};
+use tracing::{field, Span};
+use tracing_subscriber::{filter::filter_fn, prelude::*, EnvFilter};
 extern crate magic_crypt;
 
 extern crate dotenv;
@@ -28,68 +28,126 @@ mod client;
 mod db;
 mod email;
 mod error;
-mod external_ip;
-mod flows;
+mod flow;
 mod granular_acls;
 mod groups;
 mod jobs;
 mod js_eval;
 mod oauth2;
 mod parser;
-mod parser_py;
-mod parser_ts;
 mod resources;
 mod schedule;
 mod scripts;
 mod static_assets;
-mod tracing_init;
 mod users;
 mod utils;
 mod variables;
 mod worker;
-mod worker_flow;
 mod worker_ping;
 mod workspaces;
 
 use error::Error;
 
 pub use crate::email::EmailSender;
-use crate::{
-    db::UserDB,
-    error::to_anyhow,
-    oauth2::build_oauth_clients,
-    tracing_init::{MyMakeSpan, MyOnResponse},
-    utils::rd_string,
-};
-
-pub use crate::tracing_init::initialize_tracing;
+use crate::{db::UserDB, utils::rd_string};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 pub const DEFAULT_NUM_WORKERS: usize = 3;
 pub const DEFAULT_TIMEOUT: i32 = 300;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
-pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
+
+#[derive(Clone)]
+struct MyOnResponse {}
+
+impl<B> OnResponse<B> for MyOnResponse {
+    fn on_response(
+        self,
+        response: &Response<B>,
+        latency: std::time::Duration,
+        _span: &tracing::Span,
+    ) {
+        tracing::info!(
+            latency = %latency.as_millis(),
+            status = ?response.status(),
+            "finished processed request")
+    }
+}
+
+#[derive(Clone)]
+struct MyMakeSpan {}
+
+impl<B> MakeSpan<B> for MyMakeSpan {
+    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
+        tracing::info_span!(
+            "request",
+            method = %request.method(),
+            uri = %request.uri(),
+            version = ?request.version(),
+            username = field::Empty,
+        )
+    }
+}
+
+pub async fn initialize_tracing() -> anyhow::Result<()> {
+    //let log_level = if std::env::var("RUST_LOG").map(|x| &x == "debug")
+    let ts_base = tracing_subscriber::registry()
+        .with(
+            EnvFilter::from_default_env()
+                //.add_directive("windmill".parse()?)
+                .add_directive("runtime=trace".parse()?)
+                .add_directive("tokio=trace".parse()?),
+        )
+        .with(
+            tracing_subscriber::fmt::layer()
+                .json()
+                .flatten_event(true)
+                .with_span_list(false)
+                .with_current_span(true)
+                .with_filter(filter_fn(|meta| meta.target().starts_with("windmill"))),
+        );
+
+    if std::env::var("TOKIO_CONSOLE")
+        .map(|x| x == "true")
+        .unwrap_or(false)
+    {
+        let console_layer = console_subscriber::spawn();
+        ts_base.with(console_layer).init();
+    } else {
+        ts_base.init();
+    }
+    Ok(())
+}
 
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
+    let app_password = std::env::var("APP_USER_PASSWORD").unwrap_or_else(|_| "changeme".to_owned());
+
     db::migrate(db).await?;
+    db::setup_app_user(db, &app_password).await?;
     Ok(())
 }
 
 pub async fn connect_db() -> anyhow::Result<DB> {
     let database_url = std::env::var("DATABASE_URL")
         .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-
-    let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
-        Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
-        Err(_) => DEFAULT_MAX_CONNECTIONS,
-    };
-
-    Ok(db::connect(&database_url, max_connections).await?)
+    Ok(db::connect(&database_url).await?)
 }
 
+type BasicClientsMap = HashMap<String, BasicClient>;
+
+pub fn build_oauth_clients(base_url: &str) -> BasicClientsMap {
+    [(
+        "github".to_string(),
+        oauth2::build_gh_client(
+            &std::env::var("GITHUB_OAUTH_CLIENT_ID").unwrap_or_else(|_| "".to_string()),
+            &std::env::var("GITHUB_OAUTH_CLIENT_SECRET").unwrap_or_else(|_| "".to_string()),
+            base_url,
+        ),
+    )]
+    .into()
+}
+
+#[derive(Clone)]
 struct BaseUrl(String);
-struct IsSecure(bool);
-struct CloudHosted(bool);
 
 pub async fn run_server(
     db: DB,
@@ -103,16 +161,13 @@ pub async fn run_server(
     let auth_cache = Arc::new(users::AuthCache::new(db.clone()));
     let argon2 = Arc::new(Argon2::default());
     let email_sender = Arc::new(es);
-    let basic_clients = Arc::new(build_oauth_clients(base_url).await?);
+    let basic_clients = Arc::new(build_oauth_clients(base_url));
     let slack_verifier = Arc::new(
         std::env::var("SLACK_SIGNING_SECRET")
             .ok()
             .map(|x| SlackVerifier::new(x).unwrap()),
     );
-    let http_client = reqwest::ClientBuilder::new()
-        .user_agent("windmill/beta")
-        .build()
-        .map_err(to_anyhow)?;
+
     let middleware_stack = ServiceBuilder::new()
         .layer(
             TraceLayer::new_for_http()
@@ -124,14 +179,7 @@ pub async fn run_server(
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(Extension(basic_clients))
-        .layer(Extension(Arc::new(BaseUrl(base_url.to_string()))))
-        .layer(Extension(Arc::new(CloudHosted(
-            std::env::var("CLOUD_HOSTED").is_ok(),
-        ))))
-        .layer(Extension(Arc::new(IsSecure(
-            base_url.starts_with("https://"),
-        ))))
-        .layer(Extension(http_client))
+        .layer(Extension(BaseUrl(base_url.to_string())))
         .layer(CookieManagerLayer::new());
     // build our application with a route
     let app = Router::new()
@@ -157,7 +205,7 @@ pub async fn run_server(
                         .nest("/audit", audit::workspaced_service())
                         .nest("/acls", granular_acls::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/flows", flows::workspaced_service()),
+                        .nest("/flows", flow::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
@@ -166,10 +214,9 @@ pub async fn run_server(
                 )
                 .nest("/workers", worker_ping::global_service())
                 .nest("/scripts", scripts::global_service())
-                .nest("/flows", flows::global_service())
                 .nest("/schedules", schedule::global_service())
-                .route_layer(from_extractor::<users::Authed>())
-                .route_layer(from_extractor::<users::Tokened>())
+                .route_layer(extractor_middleware::<users::Authed>())
+                .route_layer(extractor_middleware::<users::Tokened>())
                 .nest(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),
@@ -200,13 +247,14 @@ pub async fn run_server(
     Ok(())
 }
 
-pub fn monitor_db(db: &DB, timeout: i32, rx: tokio::sync::broadcast::Receiver<()>) {
+pub fn monitor_db(db: &DB, timeout: i32, tx: tokio::sync::broadcast::Sender<()>) {
     let db1 = db.clone();
     let db2 = db.clone();
 
-    let rx2 = rx.resubscribe();
+    let rx1 = tx.subscribe();
+    let rx2 = tx.subscribe();
 
-    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx).await });
+    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx1).await });
     tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
@@ -217,27 +265,32 @@ pub async fn run_workers(
     num_workers: i32,
     sleep_queue: u64,
     base_url: String,
-    disable_nuser: bool,
-    disable_nsjail: bool,
-    rx: tokio::sync::broadcast::Receiver<()>,
+    tx: tokio::sync::broadcast::Sender<()>,
 ) -> anyhow::Result<()> {
     let instance_name = rd_string(5);
 
-    let ip = external_ip::get_ip().await.unwrap_or_else(|e| {
-        tracing::warn!(error = e.to_string(), "failed to get external IP");
-        "unretrievable IP".to_string()
-    });
+    let mutex = Arc::new(Mutex::new(0));
+
+    let sources: external_ip::Sources = external_ip::get_http_sources();
+    let consensus = external_ip::ConsensusBuilder::new()
+        .add_sources(sources)
+        .build();
+
+    let ip = consensus
+        .get_consensus()
+        .await
+        .map(|x| x.to_string())
+        .unwrap_or_else(|| "Unretrievable ip".to_string());
 
     let mut handles = Vec::new();
-    let mut rng = rand::thread_rng();
-
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
+        let m1 = mutex.clone();
         let ip = ip.clone();
+        let tx = tx.clone();
         let base_url = base_url.clone();
-        let rx = rx.resubscribe();
         handles.push(tokio::spawn(async move {
             tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
             worker::run_worker(
@@ -247,20 +300,14 @@ pub async fn run_workers(
                 worker_name,
                 i as u64,
                 num_workers as u64,
+                m1,
                 &ip,
                 sleep_queue,
                 &base_url,
-                disable_nuser,
-                disable_nsjail,
-                rx,
+                tx,
             )
             .await
         }));
-        //avoid having all the workers start at the same time
-        tokio::time::sleep(std::time::Duration::from_millis(
-            rng.gen_range(0..sleep_queue * num_workers as u64),
-        ))
-        .await;
     }
     futures::future::try_join_all(handles).await?;
     Ok(())
@@ -293,24 +340,3 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
     let _ = tx.send(());
     Ok(())
 }
-
-pub async fn serve_metrics(
-    addr: SocketAddr,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-) -> Result<(), hyper::Error> {
-    axum::Server::bind(&addr)
-        .serve(
-            Router::new()
-                .route("/metrics", get(metrics))
-                .into_make_service(),
-        )
-        .with_graceful_shutdown(rx.recv().map(drop))
-        .await
-}
-
-async fn metrics() -> Result<String, Error> {
-    let metric_families = prometheus::gather();
-    Ok(prometheus::TextEncoder::new()
-        .encode_to_string(&metric_families)
-        .map_err(anyhow::Error::from)?)
-}

backend/src/main.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -14,7 +13,7 @@ use dotenv::dotenv;
 async fn main() -> anyhow::Result<()> {
     dotenv().ok();
 
-    windmill::initialize_tracing();
+    windmill::initialize_tracing().await?;
 
     let db = windmill::connect_db().await?;
 
@@ -23,16 +22,6 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32);
 
-    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-        .ok()
-        .map(|s| {
-            s.parse::<bool>()
-                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-                .or_else(|_| s.parse::<SocketAddr>().map(Some))
-        })
-        .transpose()?
-        .flatten();
-
     let (server_mode, monitor_mode, migrate_db) = (true, true, true);
 
     if migrate_db {
@@ -40,7 +29,7 @@ async fn main() -> anyhow::Result<()> {
     }
 
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill::shutdown_signal(tx);
+    let shutdown_signal = windmill::shutdown_signal(tx.clone());
 
     if server_mode || monitor_mode || num_workers > 0 {
         let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
@@ -61,7 +50,7 @@ async fn main() -> anyhow::Result<()> {
                         server: "smtp.gmail.com".to_string(),
                         password: std::env::var("SMTP_PASSWORD").unwrap_or("NOPASS".to_string()),
                     },
-                    rx.resubscribe(),
+                    rx,
                 )
                 .await?;
             }
@@ -77,20 +66,7 @@ async fn main() -> anyhow::Result<()> {
                     .ok()
                     .and_then(|x| x.parse::<u64>().ok())
                     .unwrap_or(windmill::DEFAULT_SLEEP_QUEUE);
-                let disable_nuser = std::env::var("DISABLE_NUSER")
-                    .ok()
-                    .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
-                let disable_nsjail = std::env::var("DISABLE_NSJAIL")
-                    .ok()
-                    .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
 
-                tracing::info!(
-                    "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
-                     {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
-                     {timeout}"
-                );
                 windmill::run_workers(
                     db.clone(),
                     addr,
@@ -98,9 +74,7 @@ async fn main() -> anyhow::Result<()> {
                     num_workers,
                     sleep_queue,
                     base_url,
-                    disable_nuser,
-                    disable_nsjail,
-                    rx.resubscribe(),
+                    tx.clone(),
                 )
                 .await?;
             }
@@ -109,21 +83,12 @@ async fn main() -> anyhow::Result<()> {
 
         let monitor_f = async {
             if monitor_mode {
-                windmill::monitor_db(&db, timeout, rx.resubscribe());
+                windmill::monitor_db(&db, timeout, tx.clone());
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        let metrics_f = async {
-            match metrics_addr {
-                Some(addr) => windmill::serve_metrics(addr, rx.resubscribe())
-                    .await
-                    .map_err(anyhow::Error::from),
-                None => Ok(()),
-            }
-        };
-
-        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f, metrics_f)?;
+        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f)?;
     }
 
     Ok(())

backend/src/parser.rs

@@ -1,48 +1,592 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use serde::Serialize;
+use std::collections::HashMap;
 
-#[derive(Serialize, Debug, PartialEq)]
+use itertools::Itertools;
+use regex::Regex;
+use serde::Serialize;
+use serde_json::json;
+
+use crate::error;
+
+use rustpython_parser::{
+    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
+    parser,
+};
+#[derive(Serialize)]
 pub struct MainArgSignature {
     pub star_args: bool,
     pub star_kwargs: bool,
     pub args: Vec<Arg>,
 }
 
-#[derive(Serialize, Clone, Debug, PartialEq)]
-#[serde(rename_all(serialize = "lowercase"))]
-pub struct ObjectProperty {
-    pub key: String,
-    pub typ: Box<Typ>,
-}
-
-#[derive(Serialize, Clone, Debug, PartialEq)]
+#[derive(Serialize)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum Typ {
-    Str(Option<Vec<String>>),
+    Str,
     Int,
     Float,
     Bool,
-    List(Box<Typ>),
+    Dict,
+    List,
     Bytes,
     Datetime,
-    Resource(String),
-    Email,
-    Sql,
-    Object(Vec<ObjectProperty>),
     Unknown,
 }
 
-#[derive(Serialize, Clone, Debug, PartialEq)]
+#[derive(Serialize)]
 pub struct Arg {
     pub name: String,
     pub typ: Typ,
     pub default: Option<serde_json::Value>,
     pub has_default: bool,
 }
+
+pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
+    let ast = parser::parse_program(code)
+        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
+        .statements;
+    let param = ast.into_iter().find_map(|x| match x {
+        Located {
+            location: _,
+            node:
+                StatementType::FunctionDef {
+                    is_async: _,
+                    name,
+                    args,
+                    body: _,
+                    decorator_list: _,
+                    returns: _,
+                },
+        } if &name == "main" => Some(*args),
+        _ => None,
+    });
+    if let Some(params) = param {
+        //println!("{:?}", params);
+        let def_arg_start = params.args.len() - params.defaults.len();
+        Ok(MainArgSignature {
+            star_args: params.vararg != Varargs::None,
+            star_kwargs: params.vararg != Varargs::None,
+            args: params
+                .args
+                .into_iter()
+                .enumerate()
+                .map(|(i, x)| {
+                    let default = if i >= def_arg_start {
+                        to_value(&params.defaults[i - def_arg_start].node)
+                    } else {
+                        None
+                    };
+                    Arg {
+                        name: x.arg,
+                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
+                            Located {
+                                location: _,
+                                node: ExpressionType::Identifier { name },
+                            } => match name.as_ref() {
+                                "str" => Typ::Str,
+                                "float" => Typ::Float,
+                                "int" => Typ::Int,
+                                "bool" => Typ::Bool,
+                                "dict" => Typ::Dict,
+                                "list" => Typ::List,
+                                "bytes" => Typ::Bytes,
+                                "datetime" => Typ::Datetime,
+                                "datetime.datetime" => Typ::Datetime,
+                                _ => Typ::Unknown,
+                            },
+                            _ => Typ::Unknown,
+                        }),
+                        has_default: default.is_some(),
+                        default,
+                    }
+                })
+                .collect(),
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable".to_string(),
+        ))
+    }
+}
+
+const STDIMPORTS: [&str; 301] = [
+    "__future__",
+    "_abc",
+    "_aix_support",
+    "_ast",
+    "_asyncio",
+    "_bisect",
+    "_blake2",
+    "_bootsubprocess",
+    "_bz2",
+    "_codecs",
+    "_codecs_cn",
+    "_codecs_hk",
+    "_codecs_iso2022",
+    "_codecs_jp",
+    "_codecs_kr",
+    "_codecs_tw",
+    "_collections",
+    "_collections_abc",
+    "_compat_pickle",
+    "_compression",
+    "_contextvars",
+    "_crypt",
+    "_csv",
+    "_ctypes",
+    "_curses",
+    "_curses_panel",
+    "_datetime",
+    "_dbm",
+    "_decimal",
+    "_elementtree",
+    "_frozen_importlib",
+    "_frozen_importlib_external",
+    "_functools",
+    "_gdbm",
+    "_hashlib",
+    "_heapq",
+    "_imp",
+    "_io",
+    "_json",
+    "_locale",
+    "_lsprof",
+    "_lzma",
+    "_markupbase",
+    "_md5",
+    "_msi",
+    "_multibytecodec",
+    "_multiprocessing",
+    "_opcode",
+    "_operator",
+    "_osx_support",
+    "_overlapped",
+    "_pickle",
+    "_posixshmem",
+    "_posixsubprocess",
+    "_py_abc",
+    "_pydecimal",
+    "_pyio",
+    "_queue",
+    "_random",
+    "_sha1",
+    "_sha256",
+    "_sha3",
+    "_sha512",
+    "_signal",
+    "_sitebuiltins",
+    "_socket",
+    "_sqlite3",
+    "_sre",
+    "_ssl",
+    "_stat",
+    "_statistics",
+    "_string",
+    "_strptime",
+    "_struct",
+    "_symtable",
+    "_thread",
+    "_threading_local",
+    "_tkinter",
+    "_tracemalloc",
+    "_uuid",
+    "_warnings",
+    "_weakref",
+    "_weakrefset",
+    "_winapi",
+    "_zoneinfo",
+    "abc",
+    "aifc",
+    "antigravity",
+    "argparse",
+    "array",
+    "ast",
+    "asynchat",
+    "asyncio",
+    "asyncore",
+    "atexit",
+    "audioop",
+    "base64",
+    "bdb",
+    "binascii",
+    "binhex",
+    "bisect",
+    "builtins",
+    "bz2",
+    "cProfile",
+    "calendar",
+    "cgi",
+    "cgitb",
+    "chunk",
+    "cmath",
+    "cmd",
+    "code",
+    "codecs",
+    "codeop",
+    "collections",
+    "colorsys",
+    "compileall",
+    "concurrent",
+    "configparser",
+    "contextlib",
+    "contextvars",
+    "copy",
+    "copyreg",
+    "crypt",
+    "csv",
+    "ctypes",
+    "curses",
+    "dataclasses",
+    "datetime",
+    "dbm",
+    "decimal",
+    "difflib",
+    "dis",
+    "distutils",
+    "doctest",
+    "email",
+    "encodings",
+    "ensurepip",
+    "enum",
+    "errno",
+    "faulthandler",
+    "fcntl",
+    "filecmp",
+    "fileinput",
+    "fnmatch",
+    "fractions",
+    "ftplib",
+    "functools",
+    "gc",
+    "genericpath",
+    "getopt",
+    "getpass",
+    "gettext",
+    "glob",
+    "graphlib",
+    "grp",
+    "gzip",
+    "hashlib",
+    "heapq",
+    "hmac",
+    "html",
+    "http",
+    "idlelib",
+    "imaplib",
+    "imghdr",
+    "imp",
+    "importlib",
+    "inspect",
+    "io",
+    "ipaddress",
+    "itertools",
+    "json",
+    "keyword",
+    "lib2to3",
+    "linecache",
+    "locale",
+    "logging",
+    "lzma",
+    "mailbox",
+    "mailcap",
+    "marshal",
+    "math",
+    "mimetypes",
+    "mmap",
+    "modulefinder",
+    "msilib",
+    "msvcrt",
+    "multiprocessing",
+    "netrc",
+    "nis",
+    "nntplib",
+    "nt",
+    "ntpath",
+    "nturl2path",
+    "numbers",
+    "opcode",
+    "operator",
+    "optparse",
+    "os",
+    "ossaudiodev",
+    "pathlib",
+    "pdb",
+    "pickle",
+    "pickletools",
+    "pipes",
+    "pkgutil",
+    "platform",
+    "plistlib",
+    "poplib",
+    "posix",
+    "posixpath",
+    "pprint",
+    "profile",
+    "pstats",
+    "pty",
+    "pwd",
+    "py_compile",
+    "pyclbr",
+    "pydoc",
+    "pydoc_data",
+    "pyexpat",
+    "queue",
+    "quopri",
+    "random",
+    "re",
+    "readline",
+    "reprlib",
+    "resource",
+    "rlcompleter",
+    "runpy",
+    "sched",
+    "secrets",
+    "select",
+    "selectors",
+    "shelve",
+    "shlex",
+    "shutil",
+    "signal",
+    "site",
+    "smtpd",
+    "smtplib",
+    "sndhdr",
+    "socket",
+    "socketserver",
+    "spwd",
+    "sqlite3",
+    "sre_compile",
+    "sre_constants",
+    "sre_parse",
+    "ssl",
+    "stat",
+    "statistics",
+    "string",
+    "stringprep",
+    "struct",
+    "subprocess",
+    "sunau",
+    "symtable",
+    "sys",
+    "sysconfig",
+    "syslog",
+    "tabnanny",
+    "tarfile",
+    "telnetlib",
+    "tempfile",
+    "termios",
+    "textwrap",
+    "this",
+    "threading",
+    "time",
+    "timeit",
+    "tkinter",
+    "token",
+    "tokenize",
+    "trace",
+    "traceback",
+    "tracemalloc",
+    "tty",
+    "turtle",
+    "turtledemo",
+    "types",
+    "typing",
+    "unicodedata",
+    "unittest",
+    "urllib",
+    "uu",
+    "uuid",
+    "venv",
+    "warnings",
+    "wave",
+    "weakref",
+    "webbrowser",
+    "winreg",
+    "winsound",
+    "wsgiref",
+    "xdrlib",
+    "xml",
+    "xmlrpc",
+    "zipapp",
+    "zipfile",
+    "zipimport",
+    "",
+];
+
+fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
+    match et {
+        ExpressionType::String {
+            value: StringGroup::Constant { value },
+        } => Some(json!(value)),
+        ExpressionType::Number { value } => match value {
+            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
+            Number::Float { value } => Some(json!(value)),
+            _ => None,
+        },
+        ExpressionType::True => Some(json!(true)),
+        ExpressionType::False => Some(json!(false)),
+
+        ExpressionType::Dict { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = k
+                        .as_ref()
+                        .and_then(|x| to_value(&x.node))
+                        .and_then(|x| match x {
+                            serde_json::Value::String(s) => Some(s),
+                            _ => None,
+                        })
+                        .unwrap_or_else(|| "no_key".to_string());
+                    (key, to_value(&v.node))
+                })
+                .collect::<HashMap<String, _>>();
+            Some(json!(v))
+        }
+        ExpressionType::List { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|x| to_value(&x.node))
+                .collect::<Vec<_>>();
+            Some(json!(v))
+        }
+        ExpressionType::None => Some(json!(null)),
+
+        ExpressionType::Call {
+            function: _,
+            args: _,
+            keywords: _,
+        } => Some(json!("<function call>")),
+
+        _ => None,
+    }
+}
+
+pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
+    let find_requirements = code
+        .lines()
+        .find_position(|x| x.starts_with("#requirements:"));
+    let re = Regex::new(r"^\#(\S+)$").unwrap();
+    if let Some((pos, _)) = find_requirements {
+        let lines = code
+            .lines()
+            .skip(pos + 1)
+            .map_while(|x| {
+                re.captures(x)
+                    .map(|x| x.get(1).unwrap().as_str().to_string())
+            })
+            .collect();
+        Ok(lines)
+    } else {
+        let ast = parser::parse_program(code)
+            .map_err(|e| {
+                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+            })?
+            .statements;
+        let imports = ast
+            .into_iter()
+            .filter_map(|x| match x {
+                Located { location: _, node } => match node {
+                    StatementType::Import { names } => Some(
+                        names
+                            .into_iter()
+                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
+                            .collect::<Vec<String>>(),
+                    ),
+                    StatementType::ImportFrom {
+                        level: _,
+                        module: Some(mod_),
+                        names: _,
+                    } => Some(vec![mod_
+                        .split('.')
+                        .next()
+                        .unwrap_or("")
+                        .to_string()
+                        .replace("_", "-")]),
+                    _ => None,
+                },
+            })
+            .flatten()
+            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+            .unique()
+            .collect();
+        Ok(imports)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+
+    #[test]
+    fn test_parse_sig() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+
+def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
+
+	print(f\"Hello World and a warm welcome especially to {name}\")
+	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
+	return {\"len\": len(name), \"splitted\": name.split() }
+
+";
+        println!("{}", serde_json::to_string(&parse_signature(code)?)?);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_imports() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+import wmill
+from zanzibar.estonie import talin
+import matplotlib.pyplot as plt
+
+def main():
+    pass
+
+";
+        let r = parse_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_imports2() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+#requirements:
+#burkina=0.4
+#nigeria
+#
+#congo
+
+import os
+import wmill
+from zanzibar.estonie import talin
+
+def main():
+    pass
+
+";
+        let r = parse_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
+
+        Ok(())
+    }
+}

backend/src/parser_py.rs

@@ -1,734 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use std::collections::HashMap;
-
-use itertools::Itertools;
-use phf::phf_map;
-use regex::Regex;
-use serde_json::json;
-
-use crate::{
-    error,
-    parser::{Arg, MainArgSignature, Typ},
-};
-
-use rustpython_parser::{
-    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
-    parser,
-};
-
-fn filter_non_main(code: &str) -> String {
-    const DEF_MAIN: &str = "def main(";
-
-    let mut filtered_code = String::new();
-    let mut code_iter = code.split("\n");
-    let mut remaining: String = String::new();
-    while let Some(line) = code_iter.next() {
-        if line.starts_with(DEF_MAIN) {
-            filtered_code += DEF_MAIN;
-            remaining += line.strip_prefix(DEF_MAIN).unwrap();
-            remaining += &code_iter.join("\n");
-            break;
-        }
-    }
-    if filtered_code.is_empty() {
-        return String::new();
-    }
-    let mut chars = remaining.chars();
-    let mut open_parens = 1;
-
-    while let Some(c) = chars.next() {
-        if c == '(' {
-            open_parens += 1;
-        } else if c == ')' {
-            open_parens -= 1;
-        }
-        filtered_code.push(c);
-        if open_parens == 0 {
-            break;
-        }
-    }
-
-    filtered_code.push_str(": return");
-    return filtered_code;
-}
-
-pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
-    let filtered_code = filter_non_main(code);
-    if filtered_code.is_empty() {
-        return Err(error::Error::BadRequest(
-            "No main function found".to_string(),
-        ));
-    }
-    let ast = parser::parse_program(&filtered_code)
-        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
-        .statements;
-    let param = ast.into_iter().find_map(|x| match x {
-        Located {
-            location: _,
-            node:
-                StatementType::FunctionDef {
-                    is_async: _,
-                    name,
-                    args,
-                    body: _,
-                    decorator_list: _,
-                    returns: _,
-                },
-        } if &name == "main" => Some(*args),
-        _ => None,
-    });
-    if let Some(params) = param {
-        //println!("{:?}", params);
-        let def_arg_start = params.args.len() - params.defaults.len();
-        Ok(MainArgSignature {
-            star_args: params.vararg != Varargs::None,
-            star_kwargs: params.vararg != Varargs::None,
-            args: params
-                .args
-                .into_iter()
-                .enumerate()
-                .map(|(i, x)| {
-                    let default = if i >= def_arg_start {
-                        to_value(&params.defaults[i - def_arg_start].node)
-                    } else {
-                        None
-                    };
-                    Arg {
-                        name: x.arg,
-                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
-                            Located { location: _, node: ExpressionType::Identifier { name } } => {
-                                match name.as_ref() {
-                                    "str" => Typ::Str(None),
-                                    "float" => Typ::Float,
-                                    "int" => Typ::Int,
-                                    "bool" => Typ::Bool,
-                                    "dict" => Typ::Object(vec![]),
-                                    "list" => Typ::List(Box::new(Typ::Str(None))),
-                                    "bytes" => Typ::Bytes,
-                                    "datetime" => Typ::Datetime,
-                                    "datetime.datetime" => Typ::Datetime,
-                                    _ => Typ::Unknown,
-                                }
-                            }
-                            _ => Typ::Unknown,
-                        }),
-                        has_default: default.is_some(),
-                        default,
-                    }
-                })
-                .collect(),
-        })
-    } else {
-        Err(error::Error::ExecutionErr(
-            "main function was not findable".to_string(),
-        ))
-    }
-}
-
-fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
-    match et {
-        ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
-        ExpressionType::Number { value } => match value {
-            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
-            Number::Float { value } => Some(json!(value)),
-            _ => None,
-        },
-        ExpressionType::True => Some(json!(true)),
-        ExpressionType::False => Some(json!(false)),
-
-        ExpressionType::Dict { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .and_then(|x| to_value(&x.node))
-                        .and_then(|x| match x {
-                            serde_json::Value::String(s) => Some(s),
-                            _ => None,
-                        })
-                        .unwrap_or_else(|| "no_key".to_string());
-                    (key, to_value(&v.node))
-                })
-                .collect::<HashMap<String, _>>();
-            Some(json!(v))
-        }
-        ExpressionType::List { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|x| to_value(&x.node))
-                .collect::<Vec<_>>();
-            Some(json!(v))
-        }
-        ExpressionType::None => Some(json!(null)),
-
-        ExpressionType::Call { function: _, args: _, keywords: _ } => {
-            Some(json!("<function call>"))
-        }
-
-        _ => None,
-    }
-}
-
-static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
-    "psycopg2" => "psycopg2-binary"
-};
-
-fn replace_import(x: String) -> String {
-    PYTHON_IMPORTS_REPLACEMENT
-        .get(&x)
-        .map(|x| x.to_owned())
-        .unwrap_or(&x)
-        .to_string()
-}
-
-pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
-    let find_requirements = code
-        .lines()
-        .find_position(|x| x.starts_with("#requirements:"));
-    let re = Regex::new(r"^\#(\S+)$").unwrap();
-    if let Some((pos, _)) = find_requirements {
-        let lines = code
-            .lines()
-            .skip(pos + 1)
-            .map_while(|x| {
-                re.captures(x)
-                    .map(|x| x.get(1).unwrap().as_str().to_string())
-            })
-            .collect();
-        Ok(lines)
-    } else {
-        let code = &code
-            .split("\n")
-            .filter(|x| x.starts_with("import ") || x.starts_with("from "))
-            .join("\n");
-        let ast = parser::parse_program(code)
-            .map_err(|e| {
-                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
-            })?
-            .statements;
-
-        let imports = ast
-            .into_iter()
-            .filter_map(|x| match x {
-                Located { location: _, node } => match node {
-                    StatementType::Import { names } => Some(
-                        names
-                            .into_iter()
-                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
-                            .map(replace_import)
-                            .collect::<Vec<String>>(),
-                    ),
-                    StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
-
-                        Some(vec![replace_import(imprt)])
-                    }
-                    _ => None,
-                },
-            })
-            .flatten()
-            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
-            .unique()
-            .collect();
-
-        Ok(imports)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    #[test]
-    fn test_parse_python_sig() -> anyhow::Result<()> {
-        let code = "
-
-import os
-
-def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
-
-	print(f\"Hello World and a warm welcome especially to {name}\")
-	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
-	return {\"len\": len(name), \"splitted\": name.split() }
-
-";
-        //println!("{}", serde_json::to_string()?);
-        assert_eq!(
-            parse_python_signature(code)?,
-            MainArgSignature {
-                star_args: false,
-                star_kwargs: false,
-                args: vec![
-                    Arg {
-                        name: "test1".to_string(),
-                        typ: Typ::Str(None),
-                        default: None,
-                        has_default: false
-                    },
-                    Arg {
-                        name: "name".to_string(),
-                        typ: Typ::Unknown,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    },
-                    Arg {
-                        name: "byte".to_string(),
-                        typ: Typ::Bytes,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    }
-                ]
-            }
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_sig_2() -> anyhow::Result<()> {
-        let code = "
-
-import os
-
-def main(test1: str,
-    name: datetime.datetime = datetime.now(),
-    byte: bytes = bytes(1)):
-
-	print(f\"Hello World and a warm welcome especially to {name}\")
-	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
-	return {\"len\": len(name), \"splitted\": name.split() }
-
-";
-        //println!("{}", serde_json::to_string()?);
-        assert_eq!(
-            parse_python_signature(code)?,
-            MainArgSignature {
-                star_args: false,
-                star_kwargs: false,
-                args: vec![
-                    Arg {
-                        name: "test1".to_string(),
-                        typ: Typ::Str(None),
-                        default: None,
-                        has_default: false
-                    },
-                    Arg {
-                        name: "name".to_string(),
-                        typ: Typ::Unknown,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    },
-                    Arg {
-                        name: "byte".to_string(),
-                        typ: Typ::Bytes,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    }
-                ]
-            }
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_sig_3() -> anyhow::Result<()> {
-        let code = "
-
-import os
-
-def main(test1: str,
-    name: datetime.datetime = datetime.now(),
-    byte: bytes = bytes(1)): return
-
-";
-        //println!("{}", serde_json::to_string()?);
-        assert_eq!(
-            parse_python_signature(code)?,
-            MainArgSignature {
-                star_args: false,
-                star_kwargs: false,
-                args: vec![
-                    Arg {
-                        name: "test1".to_string(),
-                        typ: Typ::Str(None),
-                        default: None,
-                        has_default: false
-                    },
-                    Arg {
-                        name: "name".to_string(),
-                        typ: Typ::Unknown,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    },
-                    Arg {
-                        name: "byte".to_string(),
-                        typ: Typ::Bytes,
-                        default: Some(json!("<function call>")),
-                        has_default: true
-                    }
-                ]
-            }
-        );
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-import wmill
-from zanzibar.estonie import talin
-import matplotlib.pyplot as plt
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports2() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-#requirements:
-#burkina=0.4
-#nigeria
-#
-#congo
-
-import os
-import wmill
-from zanzibar.estonie import talin
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
-
-        Ok(())
-    }
-}
-
-const STDIMPORTS: [&str; 301] = [
-    "__future__",
-    "_abc",
-    "_aix_support",
-    "_ast",
-    "_asyncio",
-    "_bisect",
-    "_blake2",
-    "_bootsubprocess",
-    "_bz2",
-    "_codecs",
-    "_codecs_cn",
-    "_codecs_hk",
-    "_codecs_iso2022",
-    "_codecs_jp",
-    "_codecs_kr",
-    "_codecs_tw",
-    "_collections",
-    "_collections_abc",
-    "_compat_pickle",
-    "_compression",
-    "_contextvars",
-    "_crypt",
-    "_csv",
-    "_ctypes",
-    "_curses",
-    "_curses_panel",
-    "_datetime",
-    "_dbm",
-    "_decimal",
-    "_elementtree",
-    "_frozen_importlib",
-    "_frozen_importlib_external",
-    "_functools",
-    "_gdbm",
-    "_hashlib",
-    "_heapq",
-    "_imp",
-    "_io",
-    "_json",
-    "_locale",
-    "_lsprof",
-    "_lzma",
-    "_markupbase",
-    "_md5",
-    "_msi",
-    "_multibytecodec",
-    "_multiprocessing",
-    "_opcode",
-    "_operator",
-    "_osx_support",
-    "_overlapped",
-    "_pickle",
-    "_posixshmem",
-    "_posixsubprocess",
-    "_py_abc",
-    "_pydecimal",
-    "_pyio",
-    "_queue",
-    "_random",
-    "_sha1",
-    "_sha256",
-    "_sha3",
-    "_sha512",
-    "_signal",
-    "_sitebuiltins",
-    "_socket",
-    "_sqlite3",
-    "_sre",
-    "_ssl",
-    "_stat",
-    "_statistics",
-    "_string",
-    "_strptime",
-    "_struct",
-    "_symtable",
-    "_thread",
-    "_threading_local",
-    "_tkinter",
-    "_tracemalloc",
-    "_uuid",
-    "_warnings",
-    "_weakref",
-    "_weakrefset",
-    "_winapi",
-    "_zoneinfo",
-    "abc",
-    "aifc",
-    "antigravity",
-    "argparse",
-    "array",
-    "ast",
-    "asynchat",
-    "asyncio",
-    "asyncore",
-    "atexit",
-    "audioop",
-    "base64",
-    "bdb",
-    "binascii",
-    "binhex",
-    "bisect",
-    "builtins",
-    "bz2",
-    "cProfile",
-    "calendar",
-    "cgi",
-    "cgitb",
-    "chunk",
-    "cmath",
-    "cmd",
-    "code",
-    "codecs",
-    "codeop",
-    "collections",
-    "colorsys",
-    "compileall",
-    "concurrent",
-    "configparser",
-    "contextlib",
-    "contextvars",
-    "copy",
-    "copyreg",
-    "crypt",
-    "csv",
-    "ctypes",
-    "curses",
-    "dataclasses",
-    "datetime",
-    "dbm",
-    "decimal",
-    "difflib",
-    "dis",
-    "distutils",
-    "doctest",
-    "email",
-    "encodings",
-    "ensurepip",
-    "enum",
-    "errno",
-    "faulthandler",
-    "fcntl",
-    "filecmp",
-    "fileinput",
-    "fnmatch",
-    "fractions",
-    "ftplib",
-    "functools",
-    "gc",
-    "genericpath",
-    "getopt",
-    "getpass",
-    "gettext",
-    "glob",
-    "graphlib",
-    "grp",
-    "gzip",
-    "hashlib",
-    "heapq",
-    "hmac",
-    "html",
-    "http",
-    "idlelib",
-    "imaplib",
-    "imghdr",
-    "imp",
-    "importlib",
-    "inspect",
-    "io",
-    "ipaddress",
-    "itertools",
-    "json",
-    "keyword",
-    "lib2to3",
-    "linecache",
-    "locale",
-    "logging",
-    "lzma",
-    "mailbox",
-    "mailcap",
-    "marshal",
-    "math",
-    "mimetypes",
-    "mmap",
-    "modulefinder",
-    "msilib",
-    "msvcrt",
-    "multiprocessing",
-    "netrc",
-    "nis",
-    "nntplib",
-    "nt",
-    "ntpath",
-    "nturl2path",
-    "numbers",
-    "opcode",
-    "operator",
-    "optparse",
-    "os",
-    "ossaudiodev",
-    "pathlib",
-    "pdb",
-    "pickle",
-    "pickletools",
-    "pipes",
-    "pkgutil",
-    "platform",
-    "plistlib",
-    "poplib",
-    "posix",
-    "posixpath",
-    "pprint",
-    "profile",
-    "pstats",
-    "pty",
-    "pwd",
-    "py_compile",
-    "pyclbr",
-    "pydoc",
-    "pydoc_data",
-    "pyexpat",
-    "queue",
-    "quopri",
-    "random",
-    "re",
-    "readline",
-    "reprlib",
-    "resource",
-    "rlcompleter",
-    "runpy",
-    "sched",
-    "secrets",
-    "select",
-    "selectors",
-    "shelve",
-    "shlex",
-    "shutil",
-    "signal",
-    "site",
-    "smtpd",
-    "smtplib",
-    "sndhdr",
-    "socket",
-    "socketserver",
-    "spwd",
-    "sqlite3",
-    "sre_compile",
-    "sre_constants",
-    "sre_parse",
-    "ssl",
-    "stat",
-    "statistics",
-    "string",
-    "stringprep",
-    "struct",
-    "subprocess",
-    "sunau",
-    "symtable",
-    "sys",
-    "sysconfig",
-    "syslog",
-    "tabnanny",
-    "tarfile",
-    "telnetlib",
-    "tempfile",
-    "termios",
-    "textwrap",
-    "this",
-    "threading",
-    "time",
-    "timeit",
-    "tkinter",
-    "token",
-    "tokenize",
-    "trace",
-    "traceback",
-    "tracemalloc",
-    "tty",
-    "turtle",
-    "turtledemo",
-    "types",
-    "typing",
-    "unicodedata",
-    "unittest",
-    "urllib",
-    "uu",
-    "uuid",
-    "venv",
-    "warnings",
-    "wave",
-    "weakref",
-    "webbrowser",
-    "winreg",
-    "winsound",
-    "wsgiref",
-    "xdrlib",
-    "xml",
-    "xmlrpc",
-    "zipapp",
-    "zipfile",
-    "zipimport",
-    "",
-];

backend/src/parser_ts.rs

@@ -1,257 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use crate::{
-    error,
-    parser::{Arg, MainArgSignature, ObjectProperty, Typ},
-};
-
-use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
-use swc_ecma_ast::{
-    AssignPat, BindingIdent, Decl, ExportDecl, Expr, FnDecl, Ident, ModuleDecl, ModuleItem, Pat,
-    Str, TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType,
-    TsOptionalType, TsPropertySignature, TsType, TsTypeElement, TsTypeLit, TsTypeRef,
-    TsUnionOrIntersectionType, TsUnionType,
-};
-use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
-
-pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
-    let cm: Lrc<SourceMap> = Default::default();
-    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
-    let lexer = Lexer::new(
-        // We want to parse ecmascript
-        Syntax::Typescript(TsConfig::default()),
-        // EsVersion defaults to es5
-        Default::default(),
-        StringInput::from(&*fm),
-        None,
-    );
-
-    let mut parser = Parser::new_from(lexer);
-
-    let mut err_s = "".to_string();
-    for e in parser.take_errors() {
-        err_s += &e.into_kind().msg().to_string();
-    }
-
-    let ast = parser
-        .parse_module()
-        .map_err(|_| {
-            error::Error::ExecutionErr(format!(
-                "Error while parsing code, it is invalid typescript"
-            ))
-        })?
-        .body;
-
-    // println!("{ast:?}");
-    let params = ast.into_iter().find_map(|x| match x {
-        ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
-            decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, function, .. }),
-            ..
-        })) if &sym.to_string() == "main" => Some(function.params),
-        _ => None,
-    });
-    if let Some(params) = params {
-        Ok(MainArgSignature {
-            star_args: false,
-            star_kwargs: false,
-            args: params
-                .into_iter()
-                .map(|x| match x.pat {
-                    Pat::Ident(ident) => {
-                        let (name, typ, nullable) = binding_ident_to_arg(&ident);
-                        Ok(Arg {
-                            name,
-                            typ,
-                            default: None,
-                            has_default: ident.id.optional || nullable,
-                        })
-                    }
-                    Pat::Assign(AssignPat { left, right, .. }) => {
-                        let (name, typ, _nullable) =
-                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
-                                error::Error::ExecutionErr(format!(
-                                    "parameter syntax unsupported: `{}`",
-                                    cm.span_to_snippet(left.span())
-                                        .unwrap_or_else(|_| cm.span_to_string(left.span()))
-                                ))
-                            })?;
-                        Ok(Arg {
-                            name,
-                            typ,
-                            default: serde_json::to_value(right)
-                                .map_err(|e| error::Error::ExecutionErr(e.to_string()))?
-                                .as_object()
-                                .and_then(|x| x.get("value").to_owned())
-                                .cloned(),
-
-                            has_default: true,
-                        })
-                    }
-                    _ => Err(error::Error::ExecutionErr(format!(
-                        "parameter syntax unsupported: `{}`",
-                        cm.span_to_snippet(x.span())
-                            .unwrap_or_else(|_| cm.span_to_string(x.span()))
-                    ))),
-                })
-                .collect::<Result<Vec<Arg>, error::Error>>()?,
-        })
-    } else {
-        Err(error::Error::ExecutionErr(
-            "main function was not findable (expected to find 'export main function(...)'"
-                .to_string(),
-        ))
-    }
-}
-
-fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
-    let (typ, nullable) = type_ann
-        .as_ref()
-        .map(|x| tstype_to_typ(&*x.type_ann))
-        .unwrap_or((Typ::Unknown, false));
-    (id.sym.to_string(), typ, nullable)
-}
-
-fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
-    //println!("{:?}", ts_type);
-    match ts_type {
-        TsType::TsKeywordType(t) => (
-            match t.kind {
-                TsKeywordTypeKind::TsObjectKeyword => Typ::Object(vec![]),
-                TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
-                TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
-                TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
-                TsKeywordTypeKind::TsStringKeyword => Typ::Str(None),
-                _ => Typ::Unknown,
-            },
-            false,
-        ),
-        TsType::TsTypeLit(TsTypeLit { members, .. }) => {
-            let properties = members
-                .into_iter()
-                .filter_map(|x| match x {
-                    TsTypeElement::TsPropertySignature(TsPropertySignature {
-                        key,
-                        type_ann,
-                        ..
-                    }) => match (*key.to_owned(), type_ann) {
-                        (Expr::Ident(Ident { sym, .. }), type_ann) => Some(ObjectProperty {
-                            key: sym.to_string(),
-                            typ: type_ann
-                                .as_ref()
-                                .map(|typ| Box::new(tstype_to_typ(&*typ.type_ann).0))
-                                .unwrap_or(Box::new(Typ::Unknown)),
-                        }),
-                        _ => None,
-                    },
-                    _ => None,
-                })
-                .collect();
-            (Typ::Object(properties), false)
-        }
-        // TODO: we can do better here and extract the inner type of array
-        TsType::TsArrayType(TsArrayType { elem_type, .. }) => {
-            (Typ::List(Box::new(tstype_to_typ(&**elem_type).0)), false)
-        }
-        TsType::TsLitType(TsLitType { lit: TsLit::Str(Str { value, .. }), .. }) => {
-            (Typ::Str(Some(vec![value.to_string()])), false)
-        }
-        TsType::TsOptionalType(TsOptionalType { type_ann, .. }) => {
-            (tstype_to_typ(type_ann).0, true)
-        }
-        TsType::TsUnionOrIntersectionType(TsUnionOrIntersectionType::TsUnionType(
-            TsUnionType { types, .. },
-        )) => {
-            if let Some(p) = if types.len() != 2 {
-                None
-            } else {
-                types.into_iter().position(|x| match **x {
-                    TsType::TsKeywordType(TsKeywordType { kind, .. }) => {
-                        kind == TsKeywordTypeKind::TsUndefinedKeyword
-                            || kind == TsKeywordTypeKind::TsNullKeyword
-                    }
-                    _ => false,
-                })
-            } {
-                let other_p = if p == 0 { 1 } else { 0 };
-                (tstype_to_typ(&types[other_p]).0, true)
-            } else {
-                let literals = types
-                    .into_iter()
-                    .map(|x| match &**x {
-                        TsType::TsLitType(TsLitType {
-                            lit: TsLit::Str(Str { value, .. }), ..
-                        }) => Some(value.to_string()),
-                        _ => None,
-                    })
-                    .collect::<Vec<_>>();
-                if literals.iter().find(|x| x.is_none()).is_some() {
-                    (Typ::Unknown, false)
-                } else {
-                    (
-                        Typ::Str(Some(literals.into_iter().filter_map(|x| x).collect())),
-                        false,
-                    )
-                }
-            }
-        }
-        TsType::TsTypeRef(TsTypeRef { type_name, type_params, .. }) => {
-            let sym = match type_name {
-                TsEntityName::Ident(Ident { sym, .. }) => sym,
-                TsEntityName::TsQualifiedName(p) => &*p.right.sym,
-            };
-            match sym.to_string().as_str() {
-                "Resource" => (
-                    Typ::Resource(
-                        type_params
-                            .as_ref()
-                            .and_then(|x| {
-                                x.params.get(0).and_then(|y| {
-                                    y.as_ts_lit_type().and_then(|z| {
-                                        z.lit.as_str().map(|a| a.to_owned().value.to_string())
-                                    })
-                                })
-                            })
-                            .unwrap_or_else(|| "unknown".to_string()),
-                    ),
-                    false,
-                ),
-                "Base64" => (Typ::Bytes, false),
-                "Email" => (Typ::Email, false),
-                "Sql" => (Typ::Sql, false),
-                _ => (Typ::Unknown, false),
-            }
-        }
-        _ => (Typ::Unknown, false),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    // Note this useful idiom: importing names from outer (for mod tests) scope.
-    use super::*;
-
-    #[test]
-    fn test_parse_deno_sig() -> anyhow::Result<()> {
-        let code = "
-
-export function main(test1?: string, test2: string = \"burkina\",
-    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], 
-    email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
-    opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
-    min_object: {a: string, b: number}) {
-    console.log(42)
-}
-
-";
-        println!("{}", serde_json::to_string(&parse_deno_signature(code)?)?);
-
-        Ok(())
-    }
-}

backend/src/resources.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -27,7 +26,6 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
         .route("/get/*path", get(get_resource))
-        .route("/exists/*path", get(exists_resource))
         .route("/get_value/*path", get(get_resource_value))
         .route("/update/*path", post(update_resource))
         .route("/delete/*path", delete(delete_resource))
@@ -35,7 +33,6 @@ pub fn workspaced_service() -> Router {
         .route("/type/list", get(list_resource_types))
         .route("/type/listnames", get(list_resource_types_names))
         .route("/type/get/:name", get(get_resource_type))
-        .route("/type/exists/:name", get(exists_resource_type))
         .route("/type/update/:name", post(update_resource_type))
         .route("/type/delete/:name", delete(delete_resource_type))
         .route("/type/create", post(create_resource_type))
@@ -70,7 +67,6 @@ pub struct Resource {
     pub description: Option<String>,
     pub resource_type: String,
     pub extra_perms: serde_json::Value,
-    pub is_oauth: bool,
 }
 
 #[derive(Deserialize)]
@@ -79,7 +75,6 @@ pub struct CreateResource {
     pub value: Option<serde_json::Value>,
     pub description: Option<String>,
     pub resource_type: String,
-    pub is_oauth: Option<bool>,
 }
 #[derive(Deserialize)]
 struct EditResource {
@@ -109,7 +104,6 @@ async fn list_resources(
             "description",
             "resource_type",
             "extra_perms",
-            "is_oauth",
         ])
         .order_by("path", true)
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -141,8 +135,7 @@ async fn get_resource(
 
     let resource_o = sqlx::query_as!(
         Resource,
-        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
         path.to_owned(),
         &w_id
     )
@@ -154,24 +147,6 @@ async fn get_resource(
     Ok(Json(resource))
 }
 
-async fn exists_resource(
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<bool> {
-    let path = path.to_path();
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 async fn get_resource_value(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -181,8 +156,7 @@ async fn get_resource_value(
     let mut tx = user_db.begin(&authed).await?;
 
     let value_o = sqlx::query_scalar!(
-        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
         path.to_owned(),
         &w_id
     )
@@ -204,14 +178,13 @@ async fn create_resource(
 
     sqlx::query!(
         "INSERT INTO resource
-            (workspace_id, path, value, description, resource_type, is_oauth)
-            VALUES ($1, $2, $3, $4, $5, $6)",
+            (workspace_id, path, value, description, resource_type)
+            VALUES ($1, $2, $3, $4, $5)",
         w_id,
         resource.path,
         resource.value,
         resource.description,
         resource.resource_type,
-        resource.is_oauth.unwrap_or(false)
     )
     .execute(&mut tx)
     .await?;
@@ -286,16 +259,10 @@ async fn update_resource(
     if let Some(ndesc) = ns.description {
         sqlb.set_str("description", ndesc);
     }
-
-    sqlb.returning("path");
-
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
-    let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
-
-    let npath = crate::utils::not_found_if_none(npath_o, "Resource", path)?;
-
+    sqlx::query(&sql).execute(&mut tx).await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -308,21 +275,16 @@ async fn update_resource(
     .await?;
     tx.commit().await?;
 
-    Ok(format!("resource {} updated (npath: {:?})", path, npath))
+    Ok(format!("resource {} updated (npath: {:?})", path, ns.path))
 }
 
 async fn list_resource_types(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<ResourceType>> {
-    let rows = sqlx::query_as!(
-        ResourceType,
-        "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER \
-         BY name",
-        &w_id
-    )
-    .fetch_all(&db)
-    .await?;
+    let rows = sqlx::query_as!(ResourceType, "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
+        .fetch_all(&db)
+        .await?;
 
     Ok(Json(rows))
 }
@@ -331,13 +293,9 @@ async fn list_resource_types_names(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
-    let rows = sqlx::query_scalar!(
-        "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') \
-         ORDER BY name",
-        &w_id
-    )
-    .fetch_all(&db)
-    .await?;
+    let rows = sqlx::query_scalar!("SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
+        .fetch_all(&db)
+        .await?;
 
     Ok(Json(rows))
 }
@@ -351,8 +309,7 @@ async fn get_resource_type(
 
     let resource_type_o = sqlx::query_as!(
         ResourceType,
-        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = \
-         'starter')",
+        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
         &name,
         &w_id
     )
@@ -364,22 +321,6 @@ async fn get_resource_type(
     Ok(Json(resource_type))
 }
 
-async fn exists_resource_type(
-    Extension(db): Extension<DB>,
-    Path((w_id, name)): Path<(String, String)>,
-) -> JsonResult<bool> {
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)",
-        name,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 async fn create_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/src/schedule.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -10,15 +9,15 @@ use std::str::FromStr;
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::{UserDB, DB},
-    error::{self, Error, JsonResult, Result},
+    db::UserDB,
+    error::{self, JsonResult, Result},
     jobs::{self, push, JobPayload},
     users::Authed,
-    utils::{get_owner_from_path, now_from_db, Pagination, StripPath},
+    utils::{get_owner_from_path, Pagination, StripPath},
 };
 use axum::{
     extract::{Extension, Path, Query},
-    routing::{delete, get, post},
+    routing::{get, post},
     Json, Router,
 };
 
@@ -31,10 +30,8 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_schedule))
         .route("/get/*path", get(get_schedule))
-        .route("/exists/*path", get(exists_schedule))
         .route("/create", post(create_schedule))
         .route("/update/*path", post(edit_schedule))
-        .route("/delete/*path", delete(delete_schedule))
         .route("/setenabled/*path", post(set_enabled))
 }
 
@@ -65,7 +62,6 @@ pub struct NewSchedule {
     pub script_path: String,
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
-    pub enabled: Option<bool>,
 }
 
 pub async fn push_scheduled_job<'c>(
@@ -76,9 +72,8 @@ pub async fn push_scheduled_job<'c>(
         .map_err(|e| error::Error::BadRequest(e.to_string()))?;
 
     let offset = Duration::minutes(schedule.offset_.into());
-    let now = now_from_db(&mut tx).await?;
     let next = sched
-        .after(&(now - offset + Duration::seconds(1)))
+        .after(&(chrono::Utc::now() - offset + Duration::seconds(1)))
         .next()
         .expect("a schedule should have a next event")
         + offset;
@@ -133,12 +128,8 @@ async fn create_schedule(
     cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
-
-    let schedule = sqlx::query_as!(
-        Schedule,
-        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
-         is_flow, args, enabled) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *",
+    let schedule = sqlx::query_as!(Schedule,
+        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, is_flow, args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
@@ -146,12 +137,10 @@ async fn create_schedule(
         &authed.username,
         ns.script_path,
         ns.is_flow,
-        ns.args,
-        ns.enabled.unwrap_or(false),
+        ns.args
     )
     .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
+    .await?;
 
     audit_log(
         &mut tx,
@@ -172,43 +161,11 @@ async fn create_schedule(
     )
     .await?;
 
-    let tx = if ns.enabled.unwrap_or(true) {
-        push_scheduled_job(tx, schedule).await?
-    } else {
-        tx
-    };
+    let tx = push_scheduled_job(tx, schedule).await?;
     tx.commit().await?;
-
     Ok(ns.path.to_string())
 }
 
-async fn check_flow_conflict<'c>(
-    tx: &mut Transaction<'c, Postgres>,
-    w_id: &str,
-    path: &str,
-    is_flow: bool,
-    script_path: &str,
-) -> error::Result<()> {
-    if path != script_path || !is_flow {
-        let exists_flow = sqlx::query_scalar!(
-            "SELECT EXISTS (SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
-            path,
-            w_id
-        )
-        .fetch_one(tx)
-        .await?
-        .unwrap_or(false);
-        if exists_flow {
-            return Err(error::Error::BadConfig(format!(
-                "If a schedule has the same path as or a flow, it must be its primary schedule \
-                 and hence can only trigger it. 
-            However the provided path is: {script_path} and is_flow is: {is_flow}",
-            )));
-        };
-    }
-    Ok(())
-}
-
 #[derive(Deserialize)]
 pub struct EditSchedule {
     pub schedule: String,
@@ -218,12 +175,9 @@ pub struct EditSchedule {
 }
 
 async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> Result<()> {
-    sqlx::query!(
-        "DELETE FROM queue WHERE schedule_path = $1 AND running = false",
-        path
-    )
-    .execute(db)
-    .await?;
+    sqlx::query!("DELETE FROM queue WHERE schedule_path = $1", path)
+        .execute(db)
+        .await?;
     Ok(())
 }
 
@@ -239,13 +193,9 @@ async fn edit_schedule(
 
     let mut tx = user_db.begin(&authed).await?;
 
-    check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?;
-
     clear_schedule(&mut tx, path).await?;
-    let schedule = sqlx::query_as!(
-        Schedule,
-        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path \
-         = $5 AND workspace_id = $6 RETURNING *",
+    let schedule = sqlx::query_as!(Schedule,
+        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
         es.schedule,
         es.script_path,
         es.is_flow,
@@ -254,8 +204,7 @@ async fn edit_schedule(
         w_id,
     )
     .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
+    .await?;
 
     if schedule.enabled {
         tx = push_scheduled_job(tx, schedule).await?;
@@ -335,24 +284,6 @@ async fn get_schedule(
     Ok(Json(schedule))
 }
 
-async fn exists_schedule(
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<bool> {
-    let path = path.to_path();
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2)",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 #[derive(Deserialize)]
 pub struct PreviewPayload {
     pub schedule: String,
@@ -424,38 +355,6 @@ pub async fn set_enabled(
     ))
 }
 
-async fn delete_schedule(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> Result<String> {
-    let path = path.to_path();
-    let mut tx = user_db.begin(&authed).await?;
-
-    sqlx::query!(
-        "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
-        path,
-        w_id
-    )
-    .execute(&mut tx)
-    .await?;
-
-    audit_log(
-        &mut tx,
-        &authed.username,
-        "schedule.delete",
-        ActionKind::Delete,
-        &w_id,
-        Some(path),
-        None,
-    )
-    .await?;
-
-    tx.commit().await?;
-
-    Ok(format!("schedule {} deleted", path))
-}
-
 fn schedule_to_user(path: &str) -> String {
     format!("schedule-{}", path.replace('/', "-"))
 }

backend/src/scripts.rs

@@ -1,25 +1,23 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use reqwest::Client;
 use serde::Deserializer;
 use sql_builder::prelude::*;
 
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{to_anyhow, Error, JsonResult, Result},
-    jobs, parser, parser_py, parser_ts,
+    error::{Error, JsonResult, Result},
+    jobs, parser,
     users::{owner_to_token_owner, truncate_token, Authed, Tokened},
-    utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
+    utils::{require_admin, Pagination, StripPath},
 };
 use axum::{
-    extract::{Extension, Host, Path, Query},
+    extract::{Extension, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -37,14 +35,7 @@ use std::{
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
 pub fn global_service() -> Router {
-    Router::new()
-        .route(
-            "/python/tojsonschema",
-            post(parse_python_code_to_jsonschema),
-        )
-        .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema))
-        .route("/hub/list", get(list_hub_scripts))
-        .route("/hub/get/*path", get(get_hub_script_by_path))
+    Router::new().route("/tojsonschema", post(parse_code_to_jsonschema))
 }
 
 pub fn workspaced_service() -> Router {
@@ -53,32 +44,12 @@ pub fn workspaced_service() -> Router {
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
         .route("/get/p/*path", get(get_script_by_path))
-        .route("/raw/p/*path", get(raw_script_by_path))
-        .route("/exists/p/*path", get(exists_script_by_path))
         .route("/archive/h/:hash", post(archive_script_by_hash))
         .route("/delete/h/:hash", post(delete_script_by_hash))
         .route("/get/h/:hash", get(get_script_by_hash))
-        .route("/raw/h/:hash", get(raw_script_by_hash))
         .route("/deployment_status/h/:hash", get(get_deployment_status))
 }
 
-#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
-#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
-pub enum ScriptLang {
-    Deno,
-    Python3,
-}
-
-impl ScriptLang {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            ScriptLang::Deno => "deno",
-            ScriptLang::Python3 => "python3",
-        }
-    }
-}
-
 #[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
@@ -142,8 +113,6 @@ pub struct Script {
     pub extra_perms: serde_json::Value,
     pub lock: Option<String>,
     pub lock_error_logs: Option<String>,
-    pub language: ScriptLang,
-    pub is_trigger: bool,
 }
 
 #[derive(Serialize, Deserialize, sqlx::Type, Debug)]
@@ -169,8 +138,6 @@ pub struct NewScript {
     pub schema: Option<Schema>,
     pub is_template: Option<bool>,
     pub lock: Option<Vec<String>>,
-    pub language: ScriptLang,
-    pub is_trigger: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -185,7 +152,6 @@ pub struct ListScriptQuery {
     pub order_by: Option<String>,
     pub order_desc: Option<bool>,
     pub is_template: Option<bool>,
-    pub is_trigger: Option<bool>,
 }
 
 async fn list_scripts(
@@ -209,14 +175,12 @@ async fn list_scripts(
             "created_by",
             "created_at",
             "archived",
-            "null as schema",
+            "schema",
             "deleted",
             "is_template",
             "extra_perms",
             "null as lock",
             "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
-            "language",
-            "is_trigger",
         ])
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -254,9 +218,6 @@ async fn list_scripts(
     if let Some(it) = &lq.is_template {
         sqlb.and_where_eq("is_template", it);
     }
-    if let Some(it) = &lq.is_trigger {
-        sqlb.and_where_eq("is_trigger", it);
-    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
@@ -265,22 +226,6 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
-async fn list_hub_scripts(
-    Authed { email, username, .. }: Authed,
-    Extension(http_client): Extension<Client>,
-    Host(host): Host,
-) -> JsonResult<serde_json::Value> {
-    let asks = list_elems_from_hub(
-        http_client,
-        "https://hub.windmill.dev/searchData?approved=true",
-        email,
-        username,
-        host,
-    )
-    .await?;
-    Ok(Json(asks))
-}
-
 fn hash_script(ns: &NewScript) -> i64 {
     let mut dh = DefaultHasher::new();
     ns.hash(&mut dh);
@@ -353,7 +298,7 @@ async fn create_script(
                 if let Some(clashing_hash) = clashing_hash_o {
                     return Err(Error::BadRequest(format!(
                         "A script with hash {} with same parent_hash has been found. However, the \
-                         lineage must be linear: no 2 scripts can have the same parent",
+                     lineage must be linear: no 2 scripts can have the same parent",
                         ScriptHash(clashing_hash)
                     )));
                 };
@@ -399,16 +344,10 @@ async fn create_script(
         .map(|v| v.1.clone())
         .unwrap_or(json!({}));
 
-    let lock = if ns.language == ScriptLang::Deno {
-        Some("".to_string())
-    } else {
-        ns.lock.as_ref().map(|x| x.join("\n"))
-    };
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
-        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
-         content, created_by, schema, is_template, extra_perms, lock, language, is_trigger) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
+        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \
+         created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
         &w_id,
         &hash.0,
         ns.path,
@@ -420,15 +359,13 @@ async fn create_script(
         ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
         ns.is_template.unwrap_or(false),
         extra_perms,
-        lock,
-        ns.language: ScriptLang,
-        ns.is_trigger.unwrap_or(false),
+        ns.lock.as_ref().map(|x| x.join("\n"))
     )
     .execute(&mut tx)
     .await?;
 
-    let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
-        let dependencies = parser_py::parse_python_imports(&ns.content)?;
+    let mut tx = if ns.lock.is_none() {
+        let dependencies = parser::parse_imports(&ns.content)?;
         let (_, tx) = jobs::push(
             tx,
             &w_id,
@@ -489,32 +426,6 @@ async fn create_script(
     Ok((StatusCode::CREATED, format!("{}", hash)))
 }
 
-pub async fn get_hub_script_by_path(
-    Authed { email, username, .. }: Authed,
-    Path(path): Path<StripPath>,
-    Extension(http_client): Extension<Client>,
-    Host(host): Host,
-) -> Result<String> {
-    let path = path
-        .to_path()
-        .strip_prefix("hub/")
-        .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
-
-    let content = http_get_from_hub(
-        http_client,
-        &format!("https://hub.windmill.dev/raw/{path}.ts"),
-        email,
-        username,
-        host,
-        true,
-    )
-    .await?
-    .text()
-    .await
-    .map_err(to_anyhow)?;
-    Ok(content)
-}
-
 async fn get_script_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -524,10 +435,8 @@ async fn get_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
-         AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
-         (workspace_id = $2 OR workspace_id = 'starter'))",
+        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND (workspace_id = $2 OR workspace_id = 'starter'))",
     )
     .bind(path)
     .bind(w_id)
@@ -539,53 +448,6 @@ async fn get_script_by_path(
     Ok(Json(script))
 }
 
-async fn raw_script_by_path(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> Result<String> {
-    let path = path
-        .to_path()
-        .strip_suffix(".ts")
-        .ok_or_else(|| Error::BadRequest("Raw script path must end with .ts".to_string()))?;
-    let mut tx = user_db.begin(&authed).await?;
-
-    let content_o = sqlx::query_scalar!(
-        "SELECT content FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
-         AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
-         (workspace_id = $2 OR workspace_id = 'starter'))",
-         path, w_id
-    )
-    .fetch_optional(&mut tx)
-    .await?;
-    tx.commit().await?;
-
-    let content = crate::utils::not_found_if_none(content_o, "Script", path)?;
-    Ok(content)
-}
-
-async fn exists_script_by_path(
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<bool> {
-    let path = path.to_path();
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND (workspace_id = $2 OR \
-         workspace_id = 'starter') AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 \
-         OR workspace_id = 'starter')))",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 async fn get_script_by_hash_internal<'c>(
     db: &mut Transaction<'c, Postgres>,
     workspace_id: &str,
@@ -615,21 +477,6 @@ async fn get_script_by_hash(
     Ok(Json(r))
 }
 
-async fn raw_script_by_hash(
-    authed: Authed,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, hash_str)): Path<(String, String)>,
-) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
-    let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| {
-        Error::BadRequest("Raw script path must end with .ts".to_string())
-    })?)?);
-    let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
-    tx.commit().await?;
-
-    Ok(r.content)
-}
-
 #[derive(FromRow, Serialize)]
 struct DeploymentStatus {
     lock: Option<String>,
@@ -641,10 +488,8 @@ async fn get_deployment_status(
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<DeploymentStatus> {
     let mut tx = user_db.begin(&authed).await?;
-    let status_o: Option<DeploymentStatus> = sqlx::query_as!(
-        DeploymentStatus,
-        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR \
-         workspace_id = 'starter')",
+    let status_o: Option<DeploymentStatus> = sqlx::query_as!(DeploymentStatus,
+        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
         hash.0,
         w_id,
     )
@@ -672,8 +517,7 @@ async fn archive_script_by_path(
         &w_id
     )
     .fetch_one(&db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
+    .await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -701,9 +545,7 @@ async fn archive_script_by_hash(
     )
     .bind(&hash.0)
     .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
-
+    .await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -729,15 +571,13 @@ async fn delete_script_by_hash(
 
     require_admin(authed.is_admin, &authed.username)?;
     let script = sqlx::query_as::<_, Script>(
-        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND \
-         workspace_id = $2RETURNING *",
+        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND workspace_id = $2\
+         RETURNING *",
     )
     .bind(&hash.0)
     .bind(&w_id)
     .fetch_one(&db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("deleting script by hash {w_id}: {e}")))?;
-
+    .await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -753,16 +593,10 @@ async fn delete_script_by_hash(
     Ok(Json(script))
 }
 
-async fn parse_python_code_to_jsonschema(
+async fn parse_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<parser::MainArgSignature> {
-    parser_py::parse_python_signature(&code).map(Json)
-}
-
-async fn parse_deno_code_to_jsonschema(
-    Json(code): Json<String>,
-) -> JsonResult<parser::MainArgSignature> {
-    parser_ts::parse_deno_signature(&code).map(Json)
+    parser::parse_signature(&code).map(Json)
 }
 
 pub fn to_i64(s: &str) -> Result<i64> {

backend/src/static_assets.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -12,7 +11,6 @@ use axum::{
     response::IntoResponse,
 };
 
-use mime_guess::mime;
 use rust_embed::RustEmbed;
 
 // static_handler is a handler that serves static files from the
@@ -47,22 +45,12 @@ fn serve_path(path: String) -> Response<BoxBody> {
         Some(content) => {
             let body = body::boxed(body::Full::from(content.data));
             let mime = mime_guess::from_path(path).first_or_octet_stream();
-            let mut res = Response::builder().header(header::CONTENT_TYPE, mime.as_ref());
-            if mime.as_ref() == mime::APPLICATION_JAVASCRIPT {
-                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
-            } else if (mime.type_(), mime.subtype()) == (mime::TEXT, mime::CSS) {
-                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
-            } else if (mime.type_()) == (mime::IMAGE) {
-                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
-            } else {
-                res = res.header(header::CACHE_CONTROL, "no-cache, no-store, must-revalidate");
-            }
-            res.body(body).unwrap()
+            Response::builder()
+                .header(header::CONTENT_TYPE, mime.as_ref())
+                .header(header::CACHE_CONTROL, "max-age=3600".to_owned())
+                .body(body)
+                .unwrap()
         }
-        None if path.as_str().starts_with("_app/") => Response::builder()
-            .status(404)
-            .body(body::boxed(body::Empty::new()))
-            .unwrap(),
         None => serve_path("200.html".to_owned()),
     }
 }

backend/src/tracing_init.rs

@@ -1,96 +0,0 @@
-use ::tracing::{field, Metadata, Span};
-use ::tracing_subscriber::{
-    filter::filter_fn,
-    fmt::{format, Layer},
-    prelude::*,
-    EnvFilter,
-};
-use hyper::Response;
-use tower_http::trace::{MakeSpan, OnResponse};
-
-#[derive(Clone)]
-pub struct MyOnResponse {}
-
-impl<B> OnResponse<B> for MyOnResponse {
-    fn on_response(
-        self,
-        response: &Response<B>,
-        latency: std::time::Duration,
-        _span: &tracing::Span,
-    ) {
-        tracing::info!(
-            latency = latency.as_millis(),
-            status = response.status().as_u16(),
-            "response"
-        )
-    }
-}
-
-#[derive(Clone)]
-pub struct MyMakeSpan {}
-
-impl<B> MakeSpan<B> for MyMakeSpan {
-    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
-        tracing::info_span!(
-            "request",
-            method = %request.method(),
-            uri = %request.uri(),
-            username = field::Empty,
-            workspace_id = field::Empty,
-            email = field::Empty,
-        )
-    }
-}
-
-fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
-    tracing_subscriber::fmt::layer()
-        .json()
-        .flatten_event(true)
-        .with_span_list(false)
-        .with_current_span(true)
-}
-
-fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::Compact>> {
-    tracing_subscriber::fmt::layer().compact()
-}
-
-fn filter_metadata(meta: &Metadata) -> bool {
-    meta.target().starts_with("windmill")
-}
-
-pub fn initialize_tracing() {
-    let tokio_console = std::env::var("TOKIO_CONSOLE")
-        .map(|x| x == "true")
-        .unwrap_or(false);
-    let json_fmt = std::env::var("JSON_FMT")
-        .map(|x| x == "true")
-        .unwrap_or(false);
-
-    let env_filter = EnvFilter::from_default_env();
-
-    let nenv_filter = if tokio_console {
-        env_filter
-            .add_directive("runtime=trace".parse().unwrap())
-            .add_directive("tokio=trace".parse().unwrap())
-    } else {
-        env_filter
-    };
-    let ts_base = tracing_subscriber::registry().with(nenv_filter);
-
-    match (json_fmt, tokio_console) {
-        (true, true) => ts_base
-            .with(json_layer().with_filter(filter_fn(filter_metadata)))
-            .with(console_subscriber::spawn())
-            .init(),
-        (true, false) => ts_base
-            .with(json_layer().with_filter(filter_fn(filter_metadata)))
-            .init(),
-        (false, true) => ts_base
-            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
-            .with(console_subscriber::spawn())
-            .init(),
-        _ => ts_base
-            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
-            .init(),
-    }
-}

backend/src/users.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -11,14 +10,13 @@ use std::{sync::Arc, time::Duration};
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{self, Error, JsonResult, Result},
-    utils::{require_admin, require_super_admin, Pagination},
-    IsSecure,
+    error::{Error, JsonResult, Result, self},
+    utils::{require_admin, require_super_admin, Pagination}
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
     async_trait,
-    extract::{Extension, FromRequest, Path, Query, RequestParts},
+    extract::{Extension, FromRequest, Path, RequestParts, Query},
     http,
     routing::{delete, get, post},
     Json, Router,
@@ -27,7 +25,7 @@ use hyper::StatusCode;
 use rand::rngs::OsRng;
 use retainer::Cache;
 use serde::{Deserialize, Serialize};
-use sqlx::FromRow;
+use sqlx::{FromRow};
 use time::OffsetDateTime;
 use tower_cookies::{Cookie, Cookies};
 use tracing::Span;
@@ -50,6 +48,8 @@ pub fn workspaced_service() -> Router {
         .route("/leave", post(leave_workspace))
 }
 
+
+
 pub fn global_service() -> Router {
     Router::new()
         .route("/email", get(get_email))
@@ -65,11 +65,13 @@ pub fn global_service() -> Router {
         .route("/tokens/create", post(create_token))
         .route("/tokens/delete/:token_prefix", delete(delete_token))
         .route("/tokens/list", get(list_tokens))
-    // .route("/list_invite_codes", get(list_invite_codes))
-    // .route("/create_invite_code", post(create_invite_code))
-    // .route("/signup", post(signup))
-    // .route("/lost_password", post(lost_password))
-    // .route("/use_magic_link", get(use_magic_link))
+        // .route("/list_invite_codes", get(list_invite_codes))
+        // .route("/create_invite_code", post(create_invite_code))
+        // .route("/signup", post(signup))
+        // .route("/lost_password", post(lost_password))
+        // .route("/use_magic_link", get(use_magic_link))
+
+
 }
 
 pub fn make_unauthed_service() -> Router {
@@ -83,22 +85,24 @@ pub struct AuthCache {
 
 impl AuthCache {
     pub fn new(db: DB) -> Self {
-        AuthCache { cache: Cache::new(), db }
+        AuthCache {
+            cache: Cache::new(),
+            db,
+        }
     }
 
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
-        let key = (
-            w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
-            token.to_string(),
-        );
+        let key = (w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string());
         let s = self.cache.get(&key).await.map(|c| c.to_owned());
         match s {
-            a @ Some(_) => a,
+            a @ Some(_) => {
+                a
+            },
             None => {
                 let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool)>(
-                    "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
-                     OR expiration IS NULL) RETURNING owner, email, super_admin",
+                    "UPDATE token SET last_used_at = $1 WHERE token = $2 AND (expiration > NOW() OR expiration IS NULL) RETURNING owner, email, super_admin",
                 )
+                .bind(chrono::Utc::now())
                 .bind(token)
                 .fetch_optional(&self.db)
                 .await
@@ -110,44 +114,40 @@ impl AuthCache {
                         match user {
                             (_, Some(email), super_admin) => {
                                 if w_id.is_some() {
-                                    let row_o = sqlx::query_as::<_, (String, bool)>(
-                                        "SELECT username, is_admin FROM usr where email = $1 AND \
-                                         workspace_id = $2",
-                                    )
-                                    .bind(&email)
-                                    .bind(&w_id.as_ref().unwrap())
-                                    .fetch_optional(&self.db)
-                                    .await
-                                    .unwrap_or(Some(("error".to_string(), false)));
+                                    let row_o = 
+                                    sqlx::query_as::<_, (String, bool)>(
+                                "SELECT username, is_admin FROM usr where email = $1 AND workspace_id = $2",
+                            )
+                            .bind(&email)
+                            .bind(&w_id.as_ref().unwrap())
+                            .fetch_optional(&self.db)
+                            .await
+                            .unwrap_or(Some(("error".to_string(), false)));
 
-                                    match row_o {
-                                        Some((username, is_admin)) => {
-                                            let groups = get_groups_for_user(
-                                                &w_id.as_ref().unwrap(),
-                                                &username,
-                                                &self.db,
-                                            )
-                                            .await
-                                            .ok()
-                                            .unwrap_or_default();
+                            match row_o {
+                                Some((username, is_admin)) => {
+                             let groups = get_groups_for_user(&w_id.as_ref().unwrap(),
+                                 &username, &self.db)
+                                .await
+                                .ok()
+                                .unwrap_or_default();
 
-                                            Some(Authed {
-                                                email: Some(email),
-                                                username,
-                                                is_admin: is_admin || super_admin,
-                                                groups,
-                                            })
-                                        }
-                                        None if super_admin || w_id.unwrap() == "starter" => {
-                                            Some(Authed {
-                                                email: Some(email.to_string()),
-                                                username: email,
-                                                is_admin: super_admin,
-                                                groups: vec![],
-                                            })
-                                        }
-                                        None => None,
-                                    }
+                                    Some(Authed {
+                                        email: Some(email),
+                                        username,
+                                        is_admin: is_admin || super_admin,
+                                        groups,
+                                    })
+                                }, 
+                                None if super_admin || w_id.unwrap() == "starter" => Some(Authed {
+                                        email: Some(email.to_string()),
+                                        username: email,
+                                        is_admin: super_admin,
+                                        groups: vec![],
+                                    }),
+                                None => None
+                            
+                            }
                                 } else {
                                     Some(Authed {
                                         email: Some(email.to_string()),
@@ -160,23 +160,21 @@ impl AuthCache {
                             (Some(owner), _, super_admin) if w_id.is_some() => {
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
-                                        let is_admin = super_admin
-                                            || sqlx::query_scalar!(
-                                                "SELECT is_admin FROM usr where username = $1 AND \
-                                                 workspace_id = $2",
-                                                name,
-                                                &w_id.as_ref().unwrap()
-                                            )
-                                            .fetch_one(&self.db)
+
+                                        let is_admin = super_admin || sqlx::query_scalar!(
+                                            "SELECT is_admin FROM usr where username = $1 AND workspace_id = $2",
+                                            name,
+                                            &w_id.as_ref().unwrap()
+                                        )
+                                        .fetch_one(&self.db)
+                                        .await
+                                        .ok()
+                                        .unwrap_or(false);
+
+                                        let groups = get_groups_for_user(&w_id.unwrap(), &name, &self.db)
                                             .await
                                             .ok()
-                                            .unwrap_or(false);
-
-                                        let groups =
-                                            get_groups_for_user(&w_id.unwrap(), &name, &self.db)
-                                                .await
-                                                .ok()
-                                                .unwrap_or_default();
+                                            .unwrap_or_default();
 
                                         Some(Authed {
                                             email: None,
@@ -219,8 +217,7 @@ impl AuthCache {
 
 async fn extract_token<B: Send>(req: &mut RequestParts<B>) -> Option<String> {
     let auth_header = req
-        .headers()
-        .get(http::header::AUTHORIZATION)
+        .headers().get(http::header::AUTHORIZATION)
         .and_then(|value| value.to_str().ok())
         .and_then(|s| s.strip_prefix("Bearer "));
 
@@ -251,14 +248,14 @@ where
             Ok(tokened.clone())
         } else {
             let token_o = extract_token(req).await;
-            if let Some(token) = token_o {
-                let tokened = Self { token };
-                req.extensions_mut().insert(tokened.clone());
-                Ok(tokened)
-            } else {
-                Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
+                if let Some(token) = token_o {
+                    let tokened = Self { token };
+                    req.extensions_mut().insert(tokened.clone());
+                    Ok(tokened)
+                } else {
+                    Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
+                }
             }
-        }
     }
 }
 
@@ -296,15 +293,9 @@ where
             };
             if let Some(token) = token_o {
                 if let Ok(Extension(cache)) = Extension::<Arc<AuthCache>>::from_request(req).await {
-                    if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
+                    if let Some(authed) = cache.get_authed(workspace_id, &token).await {
                         req.extensions_mut().insert(authed.clone());
                         Span::current().record("username", &authed.username.as_str());
-                        if let Some(email) = authed.email.clone() {
-                            Span::current().record("email", &email.as_str());
-                        }
-                        if let Some(workspace_id) = workspace_id {
-                            Span::current().record("workspace_id", &workspace_id);
-                        }
                         return Ok(authed);
                     }
                 }
@@ -323,22 +314,9 @@ pub struct User {
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub operator: bool,
     pub disabled: bool,
-    pub role: Option<String>,
+    pub role: Option<String>
 }
 
-#[derive(FromRow, Serialize)]
-pub struct Usage {
-    pub duration_ms: i64,
-    pub jobs: i64,
-    pub flows: i64,
-}
-
-#[derive(Serialize)]
-pub struct UserWithUsage {
-    #[serde(flatten)]
-    pub user: User,
-    pub usage: Usage,
-}
 
 #[derive(FromRow, Serialize)]
 pub struct GlobalUserInfo {
@@ -350,6 +328,7 @@ pub struct GlobalUserInfo {
     company: Option<String>,
 }
 
+
 #[derive(Serialize)]
 pub struct UserInfo {
     pub workspace_id: String,
@@ -361,7 +340,7 @@ pub struct UserInfo {
     pub groups: Vec<String>,
     pub operator: bool,
     pub disabled: bool,
-    pub role: Option<String>,
+    pub role: Option<String>
 }
 
 #[derive(FromRow, Serialize)]
@@ -403,7 +382,7 @@ pub struct NewUser {
     pub password: String,
     pub super_admin: bool,
     pub name: Option<String>,
-    pub company: Option<String>,
+    pub company: Option<String>
 }
 
 #[derive(Deserialize)]
@@ -454,6 +433,7 @@ pub struct Login {
     pub password: String,
 }
 
+
 #[derive(Deserialize)]
 pub struct Signup {
     pub email: String,
@@ -472,21 +452,27 @@ struct WorkspaceUsername {
     pub username: String,
 }
 
+#[derive(sqlx::Type, Serialize, Deserialize)]
+#[sqlx(type_name = "LOGIN_TYPE", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase"))]
+pub enum LoginType {
+    Password,
+    Github,
+}
+
 async fn exists_username(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
-    Json(WorkspaceUsername { username }): Json<WorkspaceUsername>,
+    Json(WorkspaceUsername { username }): Json<WorkspaceUsername>
 ) -> JsonResult<bool> {
     let mut tx = user_db.begin(&authed).await?;
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)",
-        &w_id,
-        &username
-    )
-    .fetch_one(&mut tx)
-    .await?
-    .unwrap_or(false);
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", 
+        &w_id, &username)
+        .fetch_one(&mut tx)
+        .await?
+        .unwrap_or(false);
     tx.commit().await?;
     Ok(Json(exists))
 }
@@ -494,33 +480,12 @@ async fn exists_username(
 async fn list_users(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
-) -> JsonResult<Vec<UserWithUsage>> {
+    Path(w_id): Path<String>
+) -> JsonResult<Vec<User>> {
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query(
-        "
-        SELECT usr.*, usage.*
-          FROM usr
-             , LATERAL (
-                SELECT COALESCE(SUM(duration_ms), 0)  duration_ms
-                     , COALESCE(SUM(job_kind     IN ('flow', 'flowpreview') ::int), 0)  flows
-                     , COALESCE(SUM(job_kind NOT IN ('flow', 'flowpreview') ::int), 0)  jobs
-                  FROM completed_job
-                 WHERE workspace_id = usr.workspace_id
-                   AND created_by = usr.username
-                   AND parent_job IS NULL
-                   AND now() - '2 week'::interval < created_at 
-               ) usage
-         WHERE workspace_id = $1
-         ",
-    )
-    .bind(&w_id)
-    .try_map(|row| {
-        // flatten not released yet https://github.com/launchbadge/sqlx/pull/1959
-        Ok(UserWithUsage { user: FromRow::from_row(&row)?, usage: FromRow::from_row(&row)? })
-    })
-    .fetch_all(&mut tx)
-    .await?;
+    let rows = sqlx::query_as!(User, "SELECT * from usr WHERE workspace_id = $1", &w_id)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
@@ -528,25 +493,20 @@ async fn list_users(
 async fn list_users_as_super_admin(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Query(pagination): Query<Pagination>,
+    Query(pagination): Query<Pagination>
 ) -> JsonResult<Vec<GlobalUserInfo>> {
     let mut tx = db.begin().await?;
     require_super_admin(&mut tx, authed.email).await?;
     let (per_page, offset) = crate::utils::paginate(pagination);
 
-    let rows = sqlx::query_as!(
-        GlobalUserInfo,
-        "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT \
-         $1 OFFSET $2",
-        per_page as i32,
-        offset as i32
-    )
-    .fetch_all(&mut tx)
-    .await?;
+    let rows = sqlx::query_as!(GlobalUserInfo, "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT $1 OFFSET $2", per_page as i32, offset as i32)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
+
 // async fn list_invite_codes(
 //     authed: Authed,
 //     Extension(db): Extension<DB>,
@@ -563,10 +523,11 @@ async fn list_users_as_super_admin(
 //     Ok(Json(rows))
 // }
 
+
 async fn list_usernames(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>,
+    Path(w_id): Path<String>
 ) -> JsonResult<Vec<String>> {
     let mut tx = user_db.begin(&authed).await?;
     let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id)
@@ -581,17 +542,14 @@ async fn list_invites(
     Extension(db): Extension<DB>,
 ) -> JsonResult<Vec<WorkspaceInvite>> {
     let mut tx = db.begin().await?;
-    let rows = sqlx::query_as!(
-        WorkspaceInvite,
-        "SELECT * from workspace_invite WHERE email = $1",
-        authed.email
-    )
-    .fetch_all(&mut tx)
-    .await?;
+    let rows = sqlx::query_as!(WorkspaceInvite, "SELECT * from workspace_invite WHERE email = $1", authed.email)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
+
 async fn logout(
     Tokened { token }: Tokened,
     cookies: Cookies,
@@ -646,42 +604,27 @@ async fn global_whoami(
     Extension(db): Extension<DB>,
     Authed { email, .. }: Authed,
 ) -> JsonResult<GlobalUserInfo> {
-    let user: GlobalUserInfo = sqlx::query_as!(
-        GlobalUserInfo,
-        "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE \
-         email = $1",
-        email
-    )
-    .fetch_one(&db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")))?;
-
+    let user: GlobalUserInfo = sqlx::query_as!(GlobalUserInfo, "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE email = $1", email)
+        .fetch_one(&db)
+        .await?;
     Ok(Json(user))
 }
 
-async fn get_email(Authed { email, .. }: Authed) -> Result<String> {
-    let email = email.ok_or(Error::BadRequest(
-        "current session does not correspond to an user with email".to_string(),
-    ))?;
+async fn get_email(
+    Authed { email, .. }: Authed,
+) -> Result<String> {
+    let email = email.ok_or(Error::BadRequest("current session does not correspond to an user with email".to_string()))?;
     Ok(email)
 }
 
 async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
-    let user = sqlx::query_as!(
-        User,
-        "SELECT * FROM usr where username = $1 AND workspace_id = $2",
-        username,
-        w_id
-    )
-    .fetch_optional(db)
-    .await?;
-    let is_super_admin = sqlx::query_scalar!(
-        "SELECT super_admin FROM password WHERE email = $1",
-        user.as_ref().map(|x| &x.email)
-    )
-    .fetch_optional(db)
-    .await?
-    .unwrap_or(false);
+    let user = sqlx::query_as!(User, "SELECT * FROM usr where username = $1 AND workspace_id = $2", username, w_id)
+        .fetch_optional(db)
+        .await?;
+    let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", user.as_ref().map(|x| &x.email))
+        .fetch_optional(db)
+        .await?
+        .unwrap_or(false);
     let groups = get_groups_for_user(&w_id, username, db).await?;
     Ok(user.map(|usr| UserInfo {
         groups,
@@ -693,36 +636,33 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
         created_at: usr.created_at,
         operator: usr.operator,
         disabled: usr.disabled,
-        role: usr.role,
+        role: usr.role
     }))
 }
 
 async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<Vec<String>> {
-    let groups = sqlx::query_scalar!(
-        "SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2",
-        username,
-        w_id
-    )
-    .fetch_all(db)
-    .await?;
+    let groups = sqlx::query_scalar!("SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2", username, w_id)
+        .fetch_all(db)
+        .await?;
     Ok(groups)
 }
 
-async fn whois(
-    Extension(db): Extension<DB>,
-    Path((w_id, username)): Path<(String, String)>,
-) -> JsonResult<UserInfo> {
+async fn whois(Extension(db): Extension<DB>, Path((w_id, username)): Path<(String, String)>) -> JsonResult<UserInfo> {
     let user_o = get_user(&w_id, &username, &db).await?;
     let user = crate::utils::not_found_if_none(user_o, "User", username)?;
     Ok(Json(user))
 }
 
+
+
+
 // async fn create_invite_code(
 //     Authed { email, .. }: Authed,
 //     Extension(db): Extension<DB>,
 //     Json(nu): Json<NewInviteCode>,
 // ) -> Result<(StatusCode, String)> {
 
+
 //     let mut tx = db.begin().await?;
 //     require_super_admin(&mut tx, email).await?;
 
@@ -749,6 +689,7 @@ async fn decline_invite(
     Extension(db): Extension<DB>,
     Json(nu): Json<DeclineInvite>,
 ) -> Result<(StatusCode, String)> {
+
     let mut tx = db.begin().await?;
 
     let email = email.unwrap_or("".to_string());
@@ -761,10 +702,10 @@ async fn decline_invite(
     .await?;
 
     audit_log(
-        &mut tx,
-        &email,
+         &mut tx,
+         &email,
         "users.decline_invite",
-        ActionKind::Delete,
+        ActionKind::Create,
         &nu.workspace_id,
         Some(&email),
         None,
@@ -775,10 +716,7 @@ async fn decline_invite(
     if is_admin.is_some() {
         Ok((
             StatusCode::OK,
-            format!(
-                "user {} declined invite to workspace {}",
-                &email, nu.workspace_id
-            ),
+            format!("user {} declined invite to workspace {}", &email, nu.workspace_id),
         ))
     } else {
         Err(Error::NotFound(format!("invite for {email} not found")))
@@ -791,7 +729,7 @@ async fn accept_invite(
     Json(nu): Json<AcceptInvite>,
 ) -> Result<(StatusCode, String)> {
     if &nu.username == "bot" {
-        return Err(Error::BadRequest("bot is a reserved username".to_string()));
+        return Err(Error::BadRequest("bot is a reserved username".to_string()))
     }
     let mut tx = db.begin().await?;
 
@@ -809,7 +747,7 @@ async fn accept_invite(
     }
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &nu.username,
         "users.accept_invite",
         ActionKind::Create,
@@ -823,23 +761,14 @@ async fn accept_invite(
     if is_admin.is_some() {
         Ok((
             StatusCode::CREATED,
-            format!(
-                "user {} accepted invite to workspace {}",
-                &email, nu.workspace_id
-            ),
+            format!("user {} accepted invite to workspace {}", &email, nu.workspace_id),
         ))
     } else {
         Err(Error::NotFound(format!("invite for {email} not found")))
     }
 }
 
-async fn add_user_to_workspace<'c>(
-    w_id: &str,
-    email: &str,
-    username: &str,
-    is_admin: bool,
-    mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
-) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
+async fn add_user_to_workspace<'c>(w_id: &str, email: &str, username: &str, is_admin: bool, mut tx: sqlx::Transaction<'c, sqlx::Postgres>) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
     sqlx::query!(
         "INSERT INTO usr
             (workspace_id, email, username, is_admin)
@@ -861,7 +790,7 @@ async fn add_user_to_workspace<'c>(
     .execute(&mut tx)
     .await?;
     audit_log(
-        &mut tx,
+         &mut tx,
         username,
         "users.add_to_workspace",
         ActionKind::Create,
@@ -884,7 +813,7 @@ async fn update_workspace_user(
     require_admin(is_admin, &username)?;
 
     if let Some(a) = eu.is_admin {
-        sqlx::query_scalar!(
+            sqlx::query_scalar!(
             "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3",
             a,
             &username_to_update,
@@ -919,7 +848,7 @@ async fn update_user(
     require_super_admin(&mut tx, email.clone()).await?;
 
     if let Some(sa) = eu.is_super_admin {
-        sqlx::query_scalar!(
+            sqlx::query_scalar!(
             "UPDATE password SET super_admin = $1 WHERE email = $2",
             sa,
             &email_to_update
@@ -953,8 +882,7 @@ async fn create_user(
     require_super_admin(&mut tx, email.clone()).await?;
 
     sqlx::query!(
-        "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \
-         company)
+        "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company)
     VALUES ($1, $2, $3, 'password', $4, $5, $6)",
         &nu.email,
         true,
@@ -966,6 +894,7 @@ async fn create_user(
     .execute(&mut tx)
     .await?;
 
+
     audit_log(
         &mut tx,
         &email.unwrap(),
@@ -980,6 +909,7 @@ async fn create_user(
     Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
 }
 
+
 pub fn owner_to_token_owner(user: &str, is_group: bool) -> String {
     let prefix = if is_group { 'g' } else { 'u' };
     format!("{}/{}", prefix, user)
@@ -994,6 +924,7 @@ async fn delete_user(
 
     require_admin(is_admin, &username)?;
 
+
     let email_to_delete_o = sqlx::query_scalar!(
         "SELECT email FROM usr where username = $1 AND workspace_id = $2",
         username_to_delete,
@@ -1030,24 +961,18 @@ async fn set_password(
     Json(EditPassword { password }): Json<EditPassword>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email
-        .ok_or("no_email")
-        .map_err(|e| Error::NotAuthorized(e.to_string()))?;
+    let email = email.ok_or("no_email").map_err(|e| Error::NotAuthorized(e.to_string()))?;
 
     let custom_type = sqlx::query_scalar!(
-        "SELECT login_type::TEXT FROM password WHERE email = $1",
-        &email
-    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("setting password: {e}")))?
-    .unwrap_or("".to_string());
-
+            "SELECT login_type::TEXT FROM password WHERE email = $1",
+        &email)
+        .fetch_one(&mut tx)
+        .await?
+        .unwrap_or("".to_string());
+    
     if custom_type != "password".to_string() {
-        return Err(Error::BadRequest(format!(
-            "login type for {email} is of type {custom_type}. Cannot set password."
-        )));
-    }
+        return Err(Error::BadRequest(format!("login type for {email} is of type {custom_type}. Cannot set password.")))
+    } 
 
     sqlx::query!(
         "UPDATE password SET password_hash = $1 WHERE email = $2",
@@ -1088,6 +1013,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
     Ok(password_hash)
 }
 
+
 // async fn lost_password(
 //     Extension(db): Extension<DB>,
 //     Extension(es): Extension<Arc<EmailSender>>,
@@ -1107,7 +1033,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 
 //     if !exists {
 //         return Err(Error::NotFound(format!("no user found at email {email}")))
-//     }
+//     } 
 
 //     let already = sqlx::query_scalar!(
 //             "SELECT EXISTS(SELECT 1 FROM magic_link WHERE email = $1)",
@@ -1157,6 +1083,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     }
 // }
 
+
 // async fn signup(
 //     TypedHeader(host): TypedHeader<headers::Host>,
 //     Extension(db): Extension<DB>,
@@ -1171,12 +1098,14 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 // ) -> Result<(StatusCode, String)> {
 //     let mut tx = db.begin().await?;
 
+
 //     let email = sqlx::query_scalar!(
 //             "INSERT INTO password (email, password_hash, name, company) VALUES ($1, $2, $3, $4) RETURNING email",
 //         &email, &hash_password(argon2, password)?, name, company)
 //         .fetch_optional(&mut tx)
 //         .await?;
 
+
 //     if let Some(email) = email {
 //         let tx = create_magic_link(&host.hostname(), &email, &es, tx).await?;
 //         tx.commit().await?;
@@ -1190,6 +1119,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     }
 // }
 
+
 // async fn create_magic_link<'c>(host: &str, email: &str, es: &EmailSender, mut tx: sqlx::Transaction<'c, sqlx::Postgres>) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
 //     let token = gen_token();
 
@@ -1202,7 +1132,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     )
 //     .execute(&mut tx)
 //     .await?;
-
+    
 //     let encoded_token = urlencoding::encode(&token);
 //     let encoded_email = urlencoding::encode(email);
 //     es.send_email(Message::builder()
@@ -1228,18 +1158,20 @@ async fn login(
     cookies: Cookies,
     Extension(db): Extension<DB>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
-    Extension(is_secure): Extension<Arc<IsSecure>>,
-    Json(Login { email, password }): Json<Login>,
+    Json(Login {
+        email,
+        password,
+    }): Json<Login>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
-    let email_w_h: Option<(String, String, bool)> = sqlx::query_as(
-        "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \
-         'password'",
-    )
-    .bind(&email)
-    .fetch_optional(&mut tx)
-    .await?;
+
+    let email_w_h: Option<(String,  String, bool)> = sqlx::query_as(
+            "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = 'password'",
+        )
+        .bind(&email)
+        .fetch_optional(&mut tx)
+        .await?;
 
     if let Some((email, hash, super_admin)) = email_w_h {
         let parsed_hash =
@@ -1250,8 +1182,7 @@ async fn login(
         {
             Err(Error::BadRequest("Invalid login".to_string()))
         } else {
-            let token =
-                create_session_token(&email, super_admin, &mut tx, cookies, is_secure.0).await?;
+            let token = create_session_token(&email, super_admin, &mut tx, cookies).await?;
             tx.commit().await?;
             Ok(token)
         }
@@ -1260,28 +1191,22 @@ async fn login(
     }
 }
 
-pub async fn create_session_token<'c>(
-    email: &str,
-    super_admin: bool,
-    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-    cookies: Cookies,
-    is_secure: bool,
-) -> Result<String> {
+pub async fn create_session_token<'c>(email: &str, super_admin: bool, tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, cookies: Cookies) -> Result<String> {
     let token = gen_token();
     sqlx::query!(
         "INSERT INTO token
             (token, email, label, expiration, super_admin)
-            VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)",
+            VALUES ($1, $2, $3, $4, $5)",
         token,
         email,
         "session",
-        TTL_TOKEN_DB_H.to_string(),
+        chrono::Utc::now() + chrono::Duration::hours(TTL_TOKEN_DB_H as i64),
         super_admin
     )
     .execute(tx)
     .await?;
     let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
-    cookie.set_secure(is_secure);
+    cookie.set_secure(true);
     cookie.set_path(COOKIE_PATH);
     let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc();
     expire += time::Duration::days(3);
@@ -1304,8 +1229,7 @@ pub async fn create_token_for_owner(
     db: &DB,
     w_id: &str,
     owner: &str,
-    label: &str,
-    expires_in: i32,
+    NewToken { label, expiration }: NewToken,
     username: &str,
 ) -> Result<String> {
     use rand::prelude::*;
@@ -1315,27 +1239,24 @@ pub async fn create_token_for_owner(
         .map(char::from)
         .collect();
     let mut tx = db.begin().await?;
-    let is_super_admin = username.contains('@')
-        && sqlx::query_scalar!(
-            "SELECT super_admin FROM password WHERE email = $1",
-            owner.split_once('/').map(|x| x.1).unwrap_or("")
-        )
+    let is_super_admin = username.contains('@') && sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1",
+     owner.split_once('/').map(|x| x.1).unwrap_or(""))
         .fetch_optional(&mut tx)
         .await?
         .unwrap_or(false);
 
-    let expiration = sqlx::query_scalar!(
+    sqlx::query!(
         "INSERT INTO token
             (workspace_id, token, owner, label, expiration, super_admin)
-            VALUES ($1, $2, $3, $4, now() + ($5 || ' seconds')::interval, $6) RETURNING expiration",
+            VALUES ($1, $2, $3, $4, $5, $6)",
         &w_id,
         token,
         owner,
         label,
-        expires_in.to_string(),
+        expiration,
         is_super_admin
     )
-    .fetch_one(&mut tx)
+    .execute(&mut tx)
     .await?;
     audit_log(
         &mut tx,
@@ -1346,7 +1267,7 @@ pub async fn create_token_for_owner(
         Some(&truncate_token(&token)),
         Some(
             [
-                Some(("label", label)),
+                label.as_ref().map(|label| ("label", &label[..])),
                 expiration
                     .map(|x| x.to_string())
                     .as_ref()
@@ -1364,19 +1285,16 @@ pub async fn create_token_for_owner(
 
 async fn create_token(
     Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    Authed { email,.. }: Authed,
     Json(new_token): Json<NewToken>,
 ) -> Result<(StatusCode, String)> {
     let token = gen_token();
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| {
-        error::Error::BadRequest(format!("Only users with email can create tokens"))
-    })?;
-    let is_super_admin =
-        sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
-            .fetch_optional(&mut tx)
-            .await?
-            .unwrap_or(false);
+    let email = email.ok_or_else(|| error::Error::BadRequest(format!("Only users with email can create tokens")))?;
+    let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
+        .fetch_optional(&mut tx)
+        .await?
+        .unwrap_or(false);
     sqlx::query!(
         "INSERT INTO token
             (token, email, label, expiration, super_admin)
@@ -1410,8 +1328,8 @@ async fn list_tokens(
 ) -> JsonResult<Vec<TruncatedToken>> {
     let rows = sqlx::query_as!(
         TruncatedToken,
-        "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \
-         last_used_at FROM token WHERE email = $1",
+        "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at FROM token \
+         WHERE email = $1",
         email,
     )
     .fetch_all(&db)
@@ -1425,14 +1343,10 @@ async fn delete_token(
     Path(token_prefix): Path<String>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| {
-        error::Error::BadRequest(format!("Only users with email can create tokens"))
-    })?;
+    let email = email.ok_or_else(|| error::Error::BadRequest(format!("Only users with email can create tokens")))?;
     let tokens_deleted: Vec<String> = sqlx::query_scalar(
-        "DELETE FROM token
-               WHERE email = $1
-                 AND token LIKE concat($2::text, '%')
-           RETURNING concat(substring(token for 10), '*****')",
+        "DELETE FROM token WHERE email = $1 AND
+     token LIKE concat($3, '%') RETURNING concat(substring(token for 10), '*****')",
     )
     .bind(&email)
     .bind(&token_prefix)
@@ -1474,13 +1388,13 @@ async fn leave_workspace(
     .await?;
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &username,
         "users.leave_workspace",
         ActionKind::Delete,
         &w_id,
-        None,
-        None,
+None,
+None,
     )
     .await?;
     tx.commit().await?;
@@ -1488,32 +1402,36 @@ async fn leave_workspace(
     Ok(format!("left workspace {w_id}"))
 }
 
+
 pub async fn delete_expired_items_perdiodically(
     db: &DB,
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) -> () {
     loop {
         let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-            "DELETE FROM token WHERE expiration <= now()
+            "DELETE FROM token WHERE expiration <= $1
         RETURNING concat(substring(token for 10), '*****')",
         )
+        .bind(chrono::Utc::now())
         .fetch_all(db)
         .await;
 
+
         match tokens_deleted_r {
-            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
+            Ok(tokens) => tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens),
             Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
-
+        
         let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-            "DELETE FROM magic_link WHERE expiration <= now()
+            "DELETE FROM magic_link WHERE expiration <= $1
         RETURNING concat(substring(token for 10), '*****')",
         )
+        .bind(chrono::Utc::now())
         .fetch_all(db)
         .await;
 
         match magic_links_deleted_r {
-            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
+            Ok(tokens) => tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens),
             Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
 

backend/src/utils.rs

@@ -1,20 +1,18 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use reqwest::Response;
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
 
-use crate::error::{to_anyhow, Error, Result};
+use crate::error::{Error, Result};
 
 pub const MAX_PER_PAGE: usize = 1000;
-pub const DEFAULT_PER_PAGE: usize = 100;
+pub const DEFAULT_PER_PAGE: usize = 30;
 
 #[derive(Deserialize)]
 pub struct Pagination {
@@ -22,15 +20,11 @@ pub struct Pagination {
     pub per_page: Option<usize>,
 }
 #[derive(Deserialize)]
-pub struct StripPath(pub String);
+pub struct StripPath(String);
 
 impl StripPath {
     pub fn to_path(&self) -> &str {
-        if self.0.starts_with('/') {
-            self.0.strip_prefix('/').unwrap()
-        } else {
-            &self.0
-        }
+        self.0.strip_prefix('/').unwrap()
     }
 }
 
@@ -43,8 +37,7 @@ pub async fn require_super_admin<'c>(
         email.as_ref()
     )
     .fetch_one(db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
+    .await?;
     if !is_admin {
         Err(Error::NotAuthorized(
             "This endpoint require caller to be a super admin".to_owned(),
@@ -83,15 +76,6 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
     (per_page, offset)
 }
 
-pub async fn now_from_db<'c>(
-    db: &mut Transaction<'c, Postgres>,
-) -> Result<chrono::DateTime<chrono::Utc>> {
-    Ok(sqlx::query_scalar!("SELECT now()")
-        .fetch_one(db)
-        .await?
-        .unwrap())
-}
-
 pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U) -> Result<T> {
     if let Some(o) = opt {
         Ok(o)
@@ -107,46 +91,3 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
 pub fn get_owner_from_path(path: &str) -> String {
     path.split('/').take(2).collect::<Vec<_>>().join("/")
 }
-
-pub async fn list_elems_from_hub(
-    http_client: reqwest::Client,
-    url: &str,
-    email: Option<String>,
-    username: String,
-    host: String,
-) -> Result<serde_json::Value> {
-    let rows = http_get_from_hub(http_client, url, email, username, host, false)
-        .await?
-        .json::<serde_json::Value>()
-        .await
-        .map_err(to_anyhow)?;
-    Ok(rows)
-}
-
-pub async fn http_get_from_hub(
-    http_client: reqwest::Client,
-    url: &str,
-    email: Option<String>,
-    username: String,
-    host: String,
-    plain: bool,
-) -> Result<Response> {
-    let response = http_client
-        .get(url)
-        .header(
-            "Accept",
-            if plain {
-                "text/plain"
-            } else {
-                "application/json"
-            },
-        )
-        .header("X-email", email.unwrap_or_else(|| "".to_string()))
-        .header("X-username", username)
-        .header("X-hostname", host)
-        .send()
-        .await
-        .map_err(to_anyhow)?;
-
-    Ok(response)
-}

backend/src/variables.rs

@@ -1,18 +1,14 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::sync::Arc;
-
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{Error, JsonResult, Result},
-    oauth2::{AllClients, _refresh_token},
     users::Authed,
     utils::StripPath,
 };
@@ -24,7 +20,6 @@ use axum::{
 use hyper::StatusCode;
 
 use magic_crypt::{MagicCrypt256, MagicCryptTrait};
-use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use sqlx::{FromRow, Postgres, Transaction};
 
@@ -33,7 +28,6 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_variables))
         .route("/list_contextual", get(list_contextual_variables))
         .route("/get/*path", get(get_variable))
-        .route("/exists/*path", get(exists_variable))
         .route("/update/*path", post(update_variable))
         .route("/delete/*path", delete(delete_variable))
         .route("/create", post(create_variable))
@@ -56,9 +50,6 @@ pub struct ListableVariable {
     pub is_secret: bool,
     pub description: String,
     pub extra_perms: serde_json::Value,
-    pub account: Option<i32>,
-    pub is_oauth: bool,
-    pub is_expired: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -67,8 +58,6 @@ pub struct CreateVariable {
     pub value: String,
     pub is_secret: bool,
     pub description: String,
-    pub account: Option<i32>,
-    pub is_oauth: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -85,67 +74,41 @@ pub fn get_reserved_variables(
     email: &str,
     username: &str,
     job_id: &str,
-    permissioned_as: &str,
-    path: Option<String>,
-    flow_path: Option<String>,
-    schedule_path: Option<String>,
-) -> [ContextualVariable; 9] {
+) -> [ContextualVariable; 5] {
     [
         ContextualVariable {
             name: "WM_WORKSPACE".to_string(),
             value: w_id.to_string(),
-            description: "Workspace id of the current script".to_string(),
+            description: "Workspace id of the current script".to_string()
         },
         ContextualVariable {
             name: "WM_TOKEN".to_string(),
             value: token.to_string(),
-            description: "Token ephemeral to the current script with equal permission to the \
-                          permission of the run (Usable as a bearer token)"
-                .to_string(),
+            description: "Token ephemeral to the current script with equal permission to the permission of the run (Usable as a bearer token)".to_string()
         },
         ContextualVariable {
             name: "WM_EMAIL".to_string(),
             value: email.to_string(),
-            description: "Email of the user that executed the current script".to_string(),
+            description: "Email of the user that executed the current script".to_string()
         },
         ContextualVariable {
             name: "WM_USERNAME".to_string(),
             value: username.to_string(),
-            description: "Username of the user that executed the current script".to_string(),
+            description: "Username of the user that executed the current script".to_string()
         },
         ContextualVariable {
             name: "WM_JOB_ID".to_string(),
             value: job_id.to_string(),
-            description: "Job id of the current script".to_string(),
-        },
-        ContextualVariable {
-            name: "WM_JOB_PATH".to_string(),
-            value: path.unwrap_or_else(|| "".to_string()),
-            description: "Path of the script or flow being run if any".to_string(),
-        },
-        ContextualVariable {
-            name: "WM_FLOW_PATH".to_string(),
-            value: flow_path.unwrap_or_else(|| "".to_string()),
-            description: "Path of the encapsulating flow if the job is a flow step".to_string(),
-        },
-        ContextualVariable {
-            name: "WM_SCHEDULE_PATH".to_string(),
-            value: schedule_path.unwrap_or_else(|| "".to_string()),
-            description: "Path of the schedule if the job of the step or encapsulating step has \
-                          been triggered by a schedule"
-                .to_string(),
-        },
-        ContextualVariable {
-            name: "WM_PERMISSIONED_AS".to_string(),
-            value: permissioned_as.to_string(),
-            description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
+            description: "Job id of the current script".to_string()
         },
     ]
 }
 
 async fn list_contextual_variables(
     Path(w_id): Path<String>,
-    Authed { username, email, .. }: Authed,
+    Authed {
+        username, email, ..
+    }: Authed,
 ) -> JsonResult<Vec<ContextualVariable>> {
     Ok(Json(
         get_reserved_variables(
@@ -154,10 +117,6 @@ async fn list_contextual_variables(
             &email.unwrap_or_else(|| "no email".to_string()),
             &username,
             "017e0ad5-f499-73b6-5488-92a61c5196dd",
-            format!("u/{username}").as_str(),
-            Some("u/user/script_path".to_string()),
-            Some("u/user/encapsulating_flow_path".to_string()),
-            Some("u/user/triggering_flow_path".to_string()),
         )
         .to_vec(),
     ))
@@ -171,11 +130,8 @@ async fn list_variables(
     let mut tx = user_db.begin(&authed).await?;
 
     let rows = sqlx::query_as::<_, ListableVariable>(
-        "SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as \
-         value, is_secret, description, extra_perms, account, is_oauth, false as is_expired from \
-         variable
-         WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER \
-         BY path",
+        "SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as value, is_secret, description, extra_perms from variable
+         WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER BY path",
     )
     .bind(&w_id)
     .fetch_all(&mut tx)
@@ -195,18 +151,12 @@ async fn get_variable(
     Extension(user_db): Extension<UserDB>,
     Query(q): Query<GetVariableQuery>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Extension(clients): Extension<Arc<AllClients>>,
-    Extension(http_client): Extension<Client>,
 ) -> JsonResult<ListableVariable> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
     let variable_o = sqlx::query_as::<_, ListableVariable>(
-        "SELECT variable.*, (now() > account.expires_at) as is_expired from variable
-        LEFT JOIN account ON variable.account = account.id
-        WHERE variable.path = $1 AND (variable.workspace_id = $2 OR (is_secret IS NOT TRUE AND \
-         variable.workspace_id = 'starter')) 
-        LIMIT 1",
+        "SELECT * from variable WHERE path = $1 AND (workspace_id = $2 OR (is_secret IS NOT TRUE AND workspace_id = 'starter'))",
     )
     .bind(&path)
     .bind(&w_id)
@@ -230,22 +180,8 @@ async fn get_variable(
         .await?;
         let value = variable.value.unwrap_or_else(|| "".to_string());
         ListableVariable {
-            value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-                Some(
-                    _refresh_token(
-                        tx,
-                        &variable.path,
-                        w_id,
-                        variable.account.unwrap(),
-                        clients,
-                        http_client,
-                    )
-                    .await?,
-                )
-            } else if !value.is_empty() && decrypt_secret {
+            value: if !value.is_empty() && decrypt_secret {
                 let mc = build_crypt(&mut tx, &w_id).await?;
-                tx.commit().await?;
-
                 Some(
                     mc.decrypt_base64_to_string(value)
                         .map_err(|e| Error::InternalErr(e.to_string()))?,
@@ -258,28 +194,11 @@ async fn get_variable(
     } else {
         variable
     };
+    tx.commit().await?;
 
     Ok(Json(r))
 }
 
-async fn exists_variable(
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-) -> JsonResult<bool> {
-    let path = path.to_path();
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)",
-        path,
-        w_id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(false);
-
-    Ok(Json(exists))
-}
-
 async fn create_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -290,22 +209,20 @@ async fn create_variable(
 
     let value = if variable.is_secret {
         let mc = build_crypt(&mut tx, &w_id).await?;
-        encrypt(&mc, &variable.value)
+        encrypt(&mc, variable.value)
     } else {
         variable.value
     };
 
     sqlx::query!(
         "INSERT INTO variable
-            (workspace_id, path, value, is_secret, description, account, is_oauth)
-            VALUES ($1, $2, $3, $4, $5, $6, $7)",
+            (workspace_id, path, value, is_secret, description)
+            VALUES ($1, $2, $3, $4, $5)",
         &w_id,
         variable.path,
         value,
         variable.is_secret,
-        variable.description,
-        variable.account,
-        variable.is_oauth.unwrap_or(false),
+        variable.description
     )
     .execute(&mut tx)
     .await?;
@@ -332,6 +249,7 @@ async fn create_variable(
 async fn delete_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -342,7 +260,7 @@ async fn delete_variable(
         path,
         w_id
     )
-    .execute(&mut tx)
+    .execute(&db)
     .await?;
     audit_log(
         &mut tx,
@@ -363,6 +281,7 @@ async fn delete_variable(
 async fn update_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditVariable>,
 ) -> Result<String> {
@@ -391,7 +310,7 @@ async fn update_variable(
 
         let value = if is_secret {
             let mc = build_crypt(&mut tx, &w_id).await?;
-            encrypt(&mc, &nvalue)
+            encrypt(&mc, nvalue)
         } else {
             nvalue
         };
@@ -410,14 +329,9 @@ async fn update_variable(
         }
         sqlb.set_str("is_secret", nbool);
     }
-    sqlb.returning("path");
-
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
 
-    let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
-
-    let npath = crate::utils::not_found_if_none(npath_o, "Variable", path)?;
-
+    sqlx::query(&sql).execute(&db).await?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -430,7 +344,7 @@ async fn update_variable(
     .await?;
     tx.commit().await?;
 
-    Ok(format!("variable {} updated (npath: {:?})", path, npath))
+    Ok(format!("variable {} updated (npath: {:?})", path, ns.path))
 }
 
 pub async fn build_crypt<'c>(
@@ -442,12 +356,10 @@ pub async fn build_crypt<'c>(
         w_id
     )
     .fetch_one(db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("fetching crypt key: {e}")))?;
-
+    .await?;
     Ok(magic_crypt::new_magic_crypt!(key, 256))
 }
 
-pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String {
+pub fn encrypt(mc: &MagicCrypt256, value: String) -> String {
     mc.encrypt_str_to_base64(value)
 }

backend/src/worker_flow.rs

@@ -1,742 +0,0 @@
-use std::collections::HashMap;
-
-use crate::{
-    db::DB,
-    error::{self, Error},
-    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
-    jobs::{
-        add_completed_job, add_completed_job_error, get_queued_job, postprocess_queued_job, push,
-        script_path_to_payload, JobPayload, QueuedJob,
-    },
-    js_eval::{eval_timeout, EvalCreds},
-    users::create_token_for_owner,
-    worker,
-};
-use anyhow::Context;
-use async_recursion::async_recursion;
-use futures::TryStreamExt;
-use serde::{Deserialize, Serialize};
-use serde_json::{json, Map, Value};
-use tracing::instrument;
-use uuid::Uuid;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct FlowStatus {
-    pub step: i32,
-    pub modules: Vec<FlowStatusModule>,
-    pub failure_module: FlowStatusModule,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct Iterator {
-    pub index: usize,
-    pub itered: Vec<Value>,
-    pub args: Map<String, Value>,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-#[serde(tag = "type")]
-pub enum FlowStatusModule {
-    WaitingForPriorSteps,
-    WaitingForEvent { event: String },
-    WaitingForExecutor { job: Uuid },
-    InProgress { job: Uuid, iterator: Option<Iterator>, forloop_jobs: Option<Vec<Uuid>> },
-    Success { job: Uuid, forloop_jobs: Option<Vec<Uuid>> },
-    Failure { job: Uuid, forloop_jobs: Option<Vec<Uuid>> },
-}
-
-impl FlowStatus {
-    pub fn new(f: &FlowValue) -> Self {
-        Self {
-            step: 0,
-            modules: vec![FlowStatusModule::WaitingForPriorSteps; f.modules.len()],
-            failure_module: FlowStatusModule::WaitingForPriorSteps,
-        }
-    }
-}
-
-#[async_recursion]
-#[instrument(level = "trace", skip_all)]
-pub async fn update_flow_status_after_job_completion(
-    db: &DB,
-    job: &QueuedJob,
-    success: bool,
-    result: serde_json::Value,
-    metrics: &worker::Metrics,
-) -> error::Result<()> {
-    tracing::debug!("HANDLE FLOW: {job:?} {success} {result:?}");
-
-    let mut tx = db.begin().await?;
-
-    let w_id = &job.workspace_id;
-
-    let flow = job
-        .parent_job
-        .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?;
-
-    let old_status_json = sqlx::query_scalar!(
-        "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
-        flow,
-        w_id
-    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| {
-        Error::InternalErr(format!(
-            "fetching flow status {flow} while reporting {success} {result:?}: {e}"
-        ))
-    })?
-    .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?;
-
-    let old_status = serde_json::from_value::<FlowStatus>(old_status_json)
-        .ok()
-        .ok_or_else(|| {
-            Error::InternalErr(format!("requiring status to be parsabled as FlowStatus"))
-        })?;
-
-    let skip_loop_failures = skip_loop_failures(flow, old_status.step, &mut tx)
-        .await?
-        .unwrap_or(false);
-
-    let module = usize::try_from(old_status.step)
-        .ok()
-        .and_then(|i| old_status.modules.get(i))
-        .unwrap_or(&old_status.failure_module);
-
-    let (step_counter, new_status) = match module {
-        FlowStatusModule::InProgress { iterator: Some(Iterator { index, itered, .. }), .. }
-            if *index + 1 < itered.len() && (success || skip_loop_failures) =>
-        {
-            (old_status.step, module.clone())
-        }
-        _ => {
-            let forloop_jobs = match module {
-                FlowStatusModule::InProgress { forloop_jobs, .. } => forloop_jobs.clone(),
-                _ => None,
-            };
-            if success || (forloop_jobs.is_some() && skip_loop_failures) {
-                (
-                    old_status.step + 1,
-                    FlowStatusModule::Success { job: job.id, forloop_jobs },
-                )
-            } else {
-                (
-                    old_status.step,
-                    FlowStatusModule::Failure { job: job.id, forloop_jobs },
-                )
-            }
-        }
-    };
-
-    let is_last_step = usize::try_from(step_counter)
-        .map(|i| !(..old_status.modules.len()).contains(&i))
-        .unwrap_or(true);
-
-    tracing::debug!(
-        "old status: {:#?}\n{:#?}\n{is_last_step}",
-        old_status,
-        new_status
-    );
-
-    let (stop_early_expr, skip_if_stop_early) =
-        sqlx::query_as::<_, (Option<String>, Option<bool>)>(
-            "
-            UPDATE queue
-               SET flow_status = JSONB_SET(
-                                 JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),
-                                                        ARRAY['step'], $3)
-             WHERE id = $4
-            RETURNING
-                (raw_flow->'modules'->$1->>'stop_after_if_expr'),
-                (raw_flow->'modules'->$1->>'skip_if_stopped')::bool
-            ",
-        )
-        .bind(old_status.step)
-        .bind(serde_json::json!(new_status))
-        .bind(serde_json::json!(step_counter))
-        .bind(flow)
-        .fetch_one(&mut tx)
-        .await
-        .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?;
-
-    tracing::debug!("UPDATE: {:?}", new_status);
-
-    let flow_job = get_queued_job(flow, w_id, &mut tx)
-        .await?
-        .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
-
-    let stop_early = success
-        && if let Some(expr) = stop_early_expr.clone() {
-            compute_stop_early(expr, result.clone()).await?
-        } else {
-            false
-        };
-
-    let result = match new_status {
-        FlowStatusModule::Success { forloop_jobs: Some(jobs), .. } => {
-            let results = sqlx::query_as(
-                "
-                  SELECT result
-                    FROM completed_job
-                   WHERE id = ANY($1)
-                     AND workspace_id = $2
-                     AND success = true
-                ORDER BY args->'iter'->'index'
-                    ",
-            )
-            .bind(jobs.as_slice())
-            .bind(w_id)
-            .fetch(&mut tx)
-            .map_ok(|(v,)| v)
-            .try_collect::<Vec<Value>>()
-            .await?;
-            json!(results)
-        }
-        _ => result.clone(),
-    };
-
-    let should_continue_flow = match success {
-        _ if stop_early => false,
-        _ if flow_job.canceled => false,
-        true => !is_last_step,
-        false if skip_loop_failures => !is_last_step,
-        false if has_failure_module(flow, &mut tx).await? => true,
-        false => false,
-    };
-
-    tx.commit().await?;
-
-    let done = if !should_continue_flow {
-        let logs = if flow_job.canceled {
-            "Flow job canceled".to_string()
-        } else if stop_early {
-            let stop_early_expr = stop_early_expr.unwrap();
-            format!("Flow job stopped early based on the stop_early_expr predicate: {stop_early_expr} returning true")
-        } else {
-            "Flow job completed".to_string()
-        };
-        tracing::debug!("{skip_if_stop_early:?}");
-        add_completed_job(
-            db,
-            &flow_job,
-            success,
-            stop_early && skip_if_stop_early.unwrap_or(false),
-            result.clone(),
-            logs,
-        )
-        .await?;
-        true
-    } else {
-        match handle_flow(&flow_job, db, result.clone()).await {
-            Err(err) => {
-                let _ = add_completed_job_error(
-                    db,
-                    &flow_job,
-                    "Unexpected error during flow chaining:\n".to_string(),
-                    err,
-                    metrics,
-                )
-                .await;
-                true
-            }
-            Ok(_) => false,
-        }
-    };
-
-    if done {
-        postprocess_queued_job(
-            flow_job.is_flow_step,
-            flow_job.schedule_path.clone(),
-            flow_job.script_path.clone(),
-            &w_id,
-            flow,
-            db,
-        )
-        .await?;
-
-        if flow_job.parent_job.is_some() {
-            return Ok(update_flow_status_after_job_completion(
-                db, &flow_job, success, result, metrics,
-            )
-            .await?);
-        }
-    }
-
-    Ok(())
-}
-
-async fn skip_loop_failures<'c>(
-    flow: Uuid,
-    step: i32,
-    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-) -> Result<Option<bool>, Error> {
-    sqlx::query_as(
-        "
-    SELECT (raw_flow->'modules'->$1->'value'->>'skip_failures')::bool
-      FROM queue
-     WHERE id = $2
-        ",
-    )
-    .bind(step)
-    .bind(flow)
-    .fetch_one(tx)
-    .await
-    .map(|(v,)| v)
-    .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e}")))
-}
-
-async fn has_failure_module<'c>(
-    flow: Uuid,
-    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-) -> Result<bool, Error> {
-    sqlx::query_scalar(
-        "
-    SELECT raw_flow->'failure_module' != 'null'::jsonb
-      FROM queue
-     WHERE id = $1
-        ",
-    )
-    .bind(flow)
-    .fetch_one(tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("error during retrieval of has_failure_module: {e}")))
-}
-
-async fn compute_stop_early(expr: String, result: serde_json::Value) -> error::Result<bool> {
-    match eval_timeout(expr, [("result".to_string(), result)].into(), None, vec![]).await? {
-        serde_json::Value::Bool(true) => Ok(true),
-        serde_json::Value::Bool(false) => Ok(false),
-        a @ _ => Err(Error::ExecutionErr(format!(
-            "Expected a boolean value, found: {a:?}"
-        ))),
-    }
-}
-
-pub fn init_flow_status(f: &FlowValue) -> FlowStatus {
-    FlowStatus::new(f)
-}
-
-pub async fn update_flow_status_in_progress(
-    db: &DB,
-    w_id: &str,
-    flow: Uuid,
-    job_in_progress: Uuid,
-) -> error::Result<()> {
-    let step = get_step_of_flow_status(db, flow).await?;
-    sqlx::query(&format!(
-        "UPDATE queue
-            SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
-            WHERE id = $3 AND workspace_id = $4",
-    ))
-    .bind(json!(job_in_progress.to_string()))
-    .bind(json!("InProgress"))
-    .bind(flow)
-    .bind(w_id)
-    .execute(db)
-    .await?;
-    Ok(())
-}
-
-#[instrument(level = "trace", skip_all)]
-pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<i32> {
-    let r = sqlx::query_scalar!(
-        "SELECT (flow_status->'step')::integer FROM queue WHERE id = $1",
-        id
-    )
-    .fetch_one(db)
-    .await
-    .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e}")))?
-    .ok_or_else(|| Error::InternalErr(format!("not found step")))?;
-    Ok(r)
-}
-
-#[instrument(level = "trace", skip_all)]
-async fn transform_input(
-    flow_args: &Option<serde_json::Value>,
-    last_result: serde_json::Value,
-    input_transforms: &HashMap<String, InputTransform>,
-    workspace: &str,
-    token: &str,
-    steps: Vec<String>,
-) -> anyhow::Result<Map<String, serde_json::Value>> {
-    let mut mapped = serde_json::Map::new();
-
-    for (key, val) in input_transforms.into_iter() {
-        if let InputTransform::Static { value } = val {
-            mapped.insert(key.to_string(), value.to_owned());
-        }
-    }
-
-    for (key, val) in input_transforms.into_iter() {
-        match val {
-            InputTransform::Static { value: _ } => (),
-            InputTransform::Javascript { expr } => {
-                let previous_result = last_result.clone();
-                let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
-                let v = eval_timeout(
-                    expr.to_string(),
-                    vec![
-                        ("params".to_string(), serde_json::json!(mapped)),
-                        ("previous_result".to_string(), previous_result),
-                        ("flow_input".to_string(), flow_input),
-                    ],
-                    Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
-                    steps.clone(),
-                )
-                .await
-                .map_err(|e| {
-                    Error::ExecutionErr(format!(
-                        "Error during isolated evaluation of expression `{expr}`:\n{e}"
-                    ))
-                })?;
-                mapped.insert(key.to_string(), v);
-                ()
-            }
-        }
-    }
-
-    Ok(mapped)
-}
-
-#[instrument(level = "trace", skip_all)]
-pub async fn handle_flow(
-    flow_job: &QueuedJob,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    last_result: serde_json::Value,
-) -> anyhow::Result<()> {
-    let value = flow_job
-        .raw_flow
-        .as_ref()
-        .ok_or_else(|| Error::InternalErr(format!("requiring a raw flow value")))?
-        .to_owned();
-    let flow = serde_json::from_value::<FlowValue>(value.to_owned())?;
-
-    if flow.modules.len() == 0 {
-        Err(Error::BadRequest(format!(
-            "A flow needs at least one module to run"
-        )))?;
-    }
-
-    push_next_flow_job(
-        flow_job,
-        flow,
-        flow_job.schedule_path.clone(),
-        db,
-        last_result,
-    )
-    .await?;
-    Ok(())
-}
-
-#[async_recursion]
-#[instrument(level = "trace", skip_all)]
-async fn push_next_flow_job(
-    flow_job: &QueuedJob,
-    flow: FlowValue,
-    schedule_path: Option<String>,
-    db: &sqlx::Pool<sqlx::Postgres>,
-    last_result: serde_json::Value,
-) -> anyhow::Result<()> {
-    let status: FlowStatus =
-        serde_json::from_value::<FlowStatus>(flow_job.flow_status.clone().unwrap_or_default())
-            .with_context(|| format!("parse flow status {}", flow_job.id))?;
-
-    /* `mut` because reassigned on FlowStatusModule::Failure when failure_module is Some */
-
-    let mut i = usize::try_from(status.step)
-        .with_context(|| format!("invalid module index {}", status.step))?;
-
-    let mut module: &FlowModule = flow
-        .modules
-        .get(i)
-        .with_context(|| format!("no module at index {}", status.step))?;
-
-    let mut status_module: FlowStatusModule = status
-        .modules
-        .get(i)
-        .cloned()
-        .with_context(|| format!("no status at index {}", status.step))?;
-
-    tracing::debug!(
-        "PUSH: module: {:#?}, status: {:#?}",
-        module.value,
-        status_module
-    );
-
-    if matches!(&status_module, FlowStatusModule::Success { .. }) {
-        anyhow::bail!("no job for {status_module:?}")
-    } else if matches!(&status_module, FlowStatusModule::Failure { .. }) {
-        /* To run to the failure module, call push_next_flow_job with the current step on
-         * FlowStatusModule::Failure.  This must update the step index to the end so that no
-         * subsequent steps are run after the failure module. */
-        i = flow.modules.len();
-        module = flow
-            .failure_module
-            .as_ref()
-            /* If this fails, it's a update_flow_status_in_progress shouldn't have called
-             * handle_flow to get here. */
-            .context("missing failure module")?;
-        status_module = status.failure_module.clone();
-    };
-
-    /* Don't evaluate `module.input_transforms` after iteration has begun.  Instead, args are
-     * carried through the Iterator by the InProgress variant. */
-
-    #[rustfmt::skip]
-    let compute_input_transform = !(   matches!(&module.value, FlowModuleValue::ForloopFlow { .. })
-                                    && matches!(&status_module, FlowStatusModule::InProgress { .. }));
-
-    let mut args = if compute_input_transform {
-        let steps = status
-            .modules
-            .iter()
-            .map(|x| match x {
-                FlowStatusModule::Success { job, .. } => job.to_string(),
-                _ => "invalid step status".to_string(),
-            })
-            .collect();
-
-        let token = create_token_for_owner(
-            &db,
-            &flow_job.workspace_id,
-            &flow_job.permissioned_as,
-            "transform-input",
-            10,
-            &flow_job.created_by,
-        )
-        .await?;
-
-        transform_input(
-            &flow_job.args,
-            last_result.clone(),
-            &module.input_transforms,
-            &flow_job.workspace_id,
-            &token,
-            steps,
-        )
-        .await?
-    } else {
-        Map::new()
-    };
-
-    /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
-
-    let next_loop_status: Option<NextLoopStatus> = if let FlowModuleValue::ForloopFlow {
-        iterator,
-        ..
-    } = &module.value
-    {
-        match status_module {
-            FlowStatusModule::WaitingForPriorSteps => {
-                /* Iterator is an InputTransform, evaluate it into an array. */
-                let itered = iterator
-                    .clone()
-                    .evaluate_with(|| vec![("result".to_string(), last_result.clone())])
-                    .await?
-                    .into_array()
-                    .map_err(|not_array| {
-                        Error::BadRequest(format!("Expected an array value, found: {not_array}"))
-                    })?;
-
-                let first = if let Some(first) = itered.first() {
-                    first
-                } else {
-                    /* Nothing to iterate, complete immediately and bail. */
-                    let next_step = i
-                        .checked_add(1)
-                        .filter(|i| (..flow.modules.len()).contains(i));
-
-                    let new_job = sqlx::query_as::<_, QueuedJob>(
-                        r#"
-                    UPDATE queue
-                       SET flow_status = JSONB_SET(
-                                         JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),
-                                                                ARRAY['step'], $3)
-                     WHERE id = $4
-                 RETURNING *
-                    "#,
-                    )
-                    .bind(status.step)
-                    .bind(json!(FlowStatusModule::Success {
-                        job: flow_job.id,
-                        forloop_jobs: Some(vec![])
-                    }))
-                    .bind(json!(next_step.unwrap_or(i)))
-                    .bind(flow_job.id)
-                    .fetch_one(db)
-                    .await?;
-
-                    return if next_step.is_some() {
-                        push_next_flow_job(&new_job, flow, schedule_path, db, json!([])).await
-                    } else {
-                        let success = true;
-                        let skipped = false;
-                        let logs = "Forloop completed without iteration".to_string();
-                        let _uuid =
-                            add_completed_job(db, &new_job, success, skipped, json!([]), logs)
-                                .await?;
-                        postprocess_queued_job(
-                            false,
-                            new_job.schedule_path,
-                            new_job.script_path,
-                            &new_job.workspace_id,
-                            new_job.id,
-                            db,
-                        )
-                        .await?;
-                        Ok(())
-                    };
-                };
-
-                args.insert("iter".to_string(), json!({ "index": 0, "value": first }));
-
-                Some(NextLoopStatus { index: 0, itered, forloop_jobs: vec![] })
-            }
-
-            FlowStatusModule::InProgress {
-                iterator: Some(Iterator { itered, index, args: iterator_args }),
-                forloop_jobs: Some(forloop_jobs),
-                ..
-            } => {
-                let (index, next) = index
-                    .checked_add(1)
-                    .and_then(|i| itered.get(i).map(|next| (i, next)))
-                    /* we shouldn't get here because update_flow_status_after_job_completion
-                     * should leave this state if there iteration is complete, but also it should
-                     * be reasonable to just enter a completed state instead of failing, similar to
-                     * iterating an empty list above */
-                    .with_context(|| format!("could not iterate index {index} of {itered:?}"))?;
-
-                args.extend(iterator_args);
-                args.insert("iter".to_string(), json!({ "index": index, "value": next }));
-
-                Some(NextLoopStatus { index, itered, forloop_jobs })
-            }
-
-            _ => Err(Error::BadRequest(format!(
-                "Unrecognized module status for ForloopFlow {status_module:?}"
-            )))?,
-        }
-    } else {
-        None
-    };
-
-    if matches!(&module.value, FlowModuleValue::ForloopFlow { .. }) {
-        if let Some(value) = &flow_job.args {
-            value
-                .as_object()
-                .ok_or_else(|| {
-                    Error::BadRequest(format!("Expected an object value, found: {value:?}"))
-                })
-                .map(|map| args.extend(map.clone()))?;
-        }
-    }
-
-    /* Finally, push the job into the queue */
-
-    let mut tx = db.begin().await?;
-
-    let job_payload = match &module.value {
-        FlowModuleValue::Script { path: script_path } => {
-            script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
-        }
-        FlowModuleValue::RawScript(raw_code) => {
-            let mut raw_code = raw_code.clone();
-            if raw_code.path.is_none() {
-                raw_code.path = Some(format!("{}/{}", flow_job.script_path(), status.step));
-            }
-            JobPayload::Code(raw_code)
-        }
-        FlowModuleValue::ForloopFlow { value, .. } => JobPayload::RawFlow {
-            value: (**value).clone(),
-            path: Some(format!("{}/{}", flow_job.script_path(), status.step)),
-        },
-        a @ FlowModuleValue::Flow { .. } => {
-            tracing::info!("Unrecognized module values {:?}", a);
-            Err(Error::BadRequest(format!(
-                "Unrecognized module values {:?}",
-                a
-            )))?
-        }
-    };
-
-    let (uuid, mut tx) = push(
-        tx,
-        &flow_job.workspace_id,
-        job_payload,
-        Some(args.clone()),
-        &flow_job.created_by,
-        flow_job.permissioned_as.to_owned(),
-        None,
-        schedule_path,
-        Some(flow_job.id),
-        true,
-    )
-    .await?;
-
-    let new_status =
-        if let Some(NextLoopStatus { index, itered, mut forloop_jobs }) = next_loop_status {
-            forloop_jobs.push(uuid);
-
-            FlowStatusModule::InProgress {
-                job: uuid,
-                iterator: Some(Iterator { index, itered, args }),
-                forloop_jobs: Some(forloop_jobs),
-            }
-        } else {
-            FlowStatusModule::WaitingForExecutor { job: uuid }
-        };
-
-    sqlx::query(
-        "
-        UPDATE queue
-           SET flow_status = JSONB_SET(
-                             JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),
-                                                    ARRAY['step'], $3)
-         WHERE id = $4
-          ",
-    )
-    .bind(status.step)
-    .bind(json!(new_status))
-    .bind(json!(i))
-    .bind(flow_job.id)
-    .execute(&mut tx)
-    .await?;
-
-    tx.commit().await?;
-
-    return Ok(());
-
-    /// Some state about the current/last forloop FlowStatusModule used to initialized the next
-    /// iteration's FlowStatusModule after pushing a job
-    struct NextLoopStatus {
-        index: usize,
-        itered: Vec<Value>,
-        forloop_jobs: Vec<Uuid>,
-    }
-}
-
-impl InputTransform {
-    async fn evaluate_with<F>(self, vars: F) -> anyhow::Result<Value>
-    where
-        F: FnOnce() -> Vec<(String, Value)>,
-    {
-        match self {
-            InputTransform::Static { value } => Ok(value),
-            InputTransform::Javascript { expr } => eval_timeout(expr, vars(), None, vec![]).await,
-        }
-    }
-}
-
-trait IntoArray: Sized {
-    fn into_array(self) -> Result<Vec<Value>, Self>;
-}
-
-impl IntoArray for Value {
-    fn into_array(self) -> Result<Vec<Value>, Self> {
-        match self {
-            Value::Array(array) => Ok(array),
-            not_array => Err(not_array),
-        }
-    }
-}

backend/src/worker_ping.rs

@@ -1,6 +1,5 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.

backend/src/workspaces.rs

@@ -1,33 +1,20 @@
 /*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
+ * Author & Copyright: Ruben Fiszel 2021
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
 use crate::{
-    audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{Error, JsonResult, Result},
-    flows::Flow,
-    resources::{Resource, ResourceType},
-    scripts::{Schema, Script},
-    users::{Authed, WorkspaceInvite},
-    utils::{require_admin, require_super_admin, Pagination},
-    variables::ListableVariable,
-};
-use axum::{
-    body::StreamBody,
-    extract::{Extension, Path, Query},
-    response::IntoResponse,
-    routing::{delete, get, post},
-    Json, Router,
+    users::{Authed, WorkspaceInvite}, utils::{require_admin, require_super_admin, Pagination}, audit::{audit_log, ActionKind}, scripts::{Script, Schema}, resources::{Resource, ResourceType}, flow::Flow, variables::ListableVariable,
 };
+use axum::{extract::{Extension, Path, Query}, routing::{get, post, delete}, Json, Router, response::{IntoResponse}, body::StreamBody};
 
-use hyper::{header, StatusCode};
+use hyper::{StatusCode, header};
 use serde::{Deserialize, Serialize};
-use sqlx::FromRow;
+use sqlx::{FromRow};
 use tempfile::TempDir;
 use tokio::fs::File;
 use tokio_util::io::ReaderStream;
@@ -42,16 +29,22 @@ pub fn workspaced_service() -> Router {
         .route("/get_settings", get(get_settings))
         .route("/edit_slack_command", post(edit_slack_command))
         .route("/tarball", get(tarball_workspace))
+
+
+
 }
 
 pub fn global_service() -> Router {
     Router::new()
-        .route("/list_as_superadmin", get(list_workspaces_as_super_admin))
-        .route("/list", get(list_workspaces))
-        .route("/users", get(user_workspaces))
-        .route("/create", post(create_workspace))
-        .route("/exists", post(exists_workspace))
-        .route("/exists_username", post(exists_username))
+    .route("/list_as_superadmin", get(list_workspaces_as_super_admin))
+    .route("/list", get(list_workspaces))
+    .route("/users", get(user_workspaces))
+    .route("/create", post(create_workspace))
+    .route("/exists", post(exists_workspace))
+    .route("/validate_username", post(validate_username))
+    .route("/validate_id", post(validate_id))
+
+
 }
 
 #[derive(FromRow, Serialize)]
@@ -60,8 +53,7 @@ struct Workspace {
     name: String,
     owner: String,
     domain: Option<String>,
-    deleted: bool,
-    premium: bool,
+    deleted: bool
 }
 
 #[derive(FromRow, Serialize, Debug)]
@@ -69,18 +61,20 @@ pub struct WorkspaceSettings {
     pub workspace_id: String,
     pub slack_team_id: Option<String>,
     pub slack_name: Option<String>,
-    pub slack_command_script: Option<String>,
+    pub slack_command_script: Option<String>
 }
 
+
 #[derive(sqlx::Type, Serialize, Deserialize, Debug)]
 #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")]
 pub enum WorkspaceKeyKind {
-    Cloud,
+    Cloud
 }
 
+
 #[derive(Deserialize)]
 struct EditCommandScript {
-    slack_command_script: Option<String>,
+    slack_command_script: Option<String>
 }
 #[derive(Deserialize)]
 struct CreateWorkspace {
@@ -90,13 +84,15 @@ struct CreateWorkspace {
     domain: Option<String>,
 }
 
+
 #[derive(Deserialize)]
 struct EditWorkspace {
     name: String,
     owner: String,
-    domain: Option<String>,
+    domain: Option<String>
 }
 
+
 #[derive(Serialize)]
 struct WorkspaceList {
     pub email: String,
@@ -110,6 +106,7 @@ struct UserWorkspace {
     pub username: String,
 }
 
+
 #[derive(Deserialize)]
 struct WorkspaceId {
     pub id: String,
@@ -121,12 +118,14 @@ struct ValidateUsername {
     pub username: String,
 }
 
+
 #[derive(Deserialize)]
 pub struct NewWorkspaceInvite {
     pub email: String,
     pub is_admin: bool,
 }
 
+
 async fn list_pending_invites(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -134,30 +133,26 @@ async fn list_pending_invites(
 ) -> JsonResult<Vec<WorkspaceInvite>> {
     require_admin(authed.is_admin, &authed.username)?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as!(
-        WorkspaceInvite,
-        "SELECT * from workspace_invite WHERE workspace_id = $1",
-        w_id
-    )
-    .fetch_all(&mut tx)
-    .await?;
+    let rows = sqlx::query_as!(WorkspaceInvite, "SELECT * from workspace_invite WHERE workspace_id = $1", w_id)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
+
 async fn exists_workspace(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Json(WorkspaceId { id }): Json<WorkspaceId>,
+    Json(WorkspaceId { id }): Json<WorkspaceId>
 ) -> JsonResult<bool> {
     let mut tx = user_db.begin(&authed).await?;
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)",
-        id
-    )
-    .fetch_one(&mut tx)
-    .await?
-    .unwrap_or(false);
+        "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)", 
+        id)
+        .fetch_one(&mut tx)
+        .await?
+        .unwrap_or(false);
     tx.commit().await?;
     Ok(Json(exists))
 }
@@ -168,17 +163,16 @@ async fn list_workspaces(
 ) -> JsonResult<Vec<Workspace>> {
     let mut tx = user_db.begin(&authed).await?;
     let workspaces = sqlx::query_as!(
-        Workspace,
-        "SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \
-         usr.email = $1 AND deleted = false",
-        authed.email.as_ref()
-    )
-    .fetch_all(&mut tx)
-    .await?;
+        Workspace, 
+        "SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false", 
+        authed.email.as_ref())
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(workspaces))
 }
 
+
 async fn get_settings(
     authed: Authed,
     Path(w_id): Path<String>,
@@ -186,14 +180,11 @@ async fn get_settings(
 ) -> JsonResult<WorkspaceSettings> {
     let mut tx = user_db.begin(&authed).await?;
     let settings = sqlx::query_as!(
-        WorkspaceSettings,
-        "SELECT * FROM workspace_settings WHERE workspace_id = $1",
-        &w_id
-    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("getting settings: {e}")))?;
-
+        WorkspaceSettings, 
+        "SELECT * FROM workspace_settings WHERE workspace_id = $1", 
+        &w_id)
+        .fetch_one(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(settings))
 }
@@ -203,7 +194,7 @@ async fn edit_slack_command(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Authed { is_admin, username, .. }: Authed,
-    Json(es): Json<EditCommandScript>,
+    Json(es): Json<EditCommandScript>
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
@@ -216,21 +207,15 @@ async fn edit_slack_command(
     .await?;
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &authed.username,
         "workspaces.edit_command_script",
         ActionKind::Update,
         &w_id,
-        Some(&authed.email.unwrap()),
-        Some(
-            [(
-                "script",
-                es.slack_command_script
-                    .unwrap_or("NO_SCRIPT".to_string())
-                    .as_str(),
-            )]
-            .into(),
-        ),
+Some(&authed.email.unwrap()),
+Some([
+            ("script", es.slack_command_script.unwrap_or("NO_SCRIPT".to_string()).as_str())
+        ].into()),
     )
     .await?;
     tx.commit().await?;
@@ -238,6 +223,7 @@ async fn edit_slack_command(
     Ok(format!("Edit command script {}", &w_id))
 }
 
+
 async fn list_workspaces_as_super_admin(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -249,13 +235,10 @@ async fn list_workspaces_as_super_admin(
     let (per_page, offset) = crate::utils::paginate(pagination);
 
     let workspaces = sqlx::query_as!(
-        Workspace,
-        "SELECT * FROM workspace LIMIT $1 OFFSET $2",
-        per_page as i32,
-        offset as i32
-    )
-    .fetch_all(&mut tx)
-    .await?;
+        Workspace, 
+        "SELECT * FROM workspace LIMIT $1 OFFSET $2", per_page as i32, offset as i32)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(workspaces))
 }
@@ -269,14 +252,12 @@ async fn user_workspaces(
         .map_err(|x| Error::NotAuthorized(x.to_string()))?;
     let mut tx = db.begin().await?;
     let workspaces = sqlx::query_as!(
-        UserWorkspace,
-        "SELECT workspace.id, workspace.name, usr.username
-     FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = \
-         false",
-        email
-    )
-    .fetch_all(&mut tx)
-    .await?;
+        UserWorkspace, 
+    "SELECT workspace.id, workspace.name, usr.username
+     FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false", 
+     email)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(WorkspaceList { email, workspaces }))
 }
@@ -284,10 +265,10 @@ async fn user_workspaces(
 async fn create_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Json(nw): Json<CreateWorkspace>,
+    Json(nw): Json<CreateWorkspace>
 ) -> Result<String> {
     if &nw.username == "bot" {
-        return Err(Error::BadRequest("bot is a reserved username".to_string()));
+        return Err(Error::BadRequest("bot is a reserved username".to_string()))
     }
     let mut tx = db.begin().await?;
     sqlx::query!(
@@ -314,8 +295,7 @@ async fn create_workspace(
         "INSERT INTO workspace_key
             (workspace_id, kind, key)
             VALUES ($1, 'cloud', $2)",
-        nw.id,
-        &key
+        nw.id, &key
     )
     .execute(&mut tx)
     .await?;
@@ -327,7 +307,7 @@ async fn create_workspace(
             VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), 
                 ($3, 'g/all/not_secret', $4, false, 'This item is not secret')",
         nw.id,
-        crate::variables::encrypt(&mc, "pretty secret value"),
+        crate::variables::encrypt(&mc, "pretty secret value".to_string()),
         nw.id,
         "finland does not actually exist",
     )
@@ -363,16 +343,17 @@ async fn create_workspace(
     .await?;
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &authed.username,
         "workspaces.create",
         ActionKind::Create,
         &nw.id,
-        Some(nw.name.as_str()),
+        Some(&authed.email.unwrap()),
         None,
     )
     .await?;
     tx.commit().await?;
+    
 
     Ok(format!("Created workspace {}", &nw.id))
 }
@@ -382,7 +363,7 @@ async fn edit_workspace(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Authed { is_admin, username, .. }: Authed,
-    Json(ew): Json<EditWorkspace>,
+    Json(ew): Json<EditWorkspace>
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
@@ -397,19 +378,15 @@ async fn edit_workspace(
     .await?;
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &authed.username,
         "workspaces.update",
         ActionKind::Update,
         &w_id,
-        Some(&authed.email.unwrap()),
-        Some(
-            [(
-                "domain",
-                ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str(),
-            )]
-            .into(),
-        ),
+Some(&authed.email.unwrap()),
+Some([
+            ("domain", ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str())
+        ].into()),
     )
     .await?;
     tx.commit().await?;
@@ -424,18 +401,21 @@ async fn delete_workspace(
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
-    sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", &w_id)
-        .execute(&mut tx)
-        .await?;
+    sqlx::query!(
+        "UPDATE workspace SET deleted = true WHERE id = $1",
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
 
     audit_log(
-        &mut tx,
+         &mut tx,
         &username,
         "workspaces.delete",
         ActionKind::Update,
         &w_id,
-        Some(&email.unwrap_or("noemail".to_string())),
-        None,
+Some(&email.unwrap_or("noemail".to_string())),
+None,
     )
     .await?;
     tx.commit().await?;
@@ -443,12 +423,14 @@ async fn delete_workspace(
     Ok(format!("Deleted workspace {}", &w_id))
 }
 
+
 async fn invite_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
+
     require_admin(is_admin, &username)?;
 
     let mut tx = db.begin().await?;
@@ -472,12 +454,14 @@ async fn invite_user(
     ))
 }
 
+
 async fn delete_invite(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
+
     require_admin(is_admin, &username)?;
 
     let mut tx = db.begin().await?;
@@ -500,10 +484,11 @@ async fn delete_invite(
     ))
 }
 
-async fn exists_username(
+async fn validate_username(
     Extension(db): Extension<DB>,
     Json(vu): Json<ValidateUsername>,
 ) -> Result<String> {
+
     let exists = sqlx::query_scalar!(
         "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)",
         vu.username,
@@ -514,79 +499,88 @@ async fn exists_username(
     .unwrap_or(true);
 
     if exists {
-        return Err(Error::BadRequest("username already taken".to_string()));
+        return Err(Error::BadRequest("username already taken".to_string()))
     }
 
     Ok("valid username".to_string())
 }
 
+
+async fn validate_id(
+    Extension(db): Extension<DB>,
+    Json(wi): Json<WorkspaceId>,
+) -> Result<String> {
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)",
+        wi.id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(true);
+
+    if exists {
+        return Err(Error::BadRequest("id already taken".to_string()))
+    }
+
+    Ok("valid workspace".to_string())
+}
+
 #[derive(Serialize)]
 struct ScriptMetadata {
     summary: String,
     description: String,
     schema: Option<Schema>,
     is_template: bool,
-    lock: Vec<String>,
+    lock: Vec<String>
 }
 
 async fn tarball_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
+) ->  Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
     let tmp_dir = TempDir::new_in(".")?;
-
+    
     let name = format!("windmill-{w_id}.tar");
     let file_path = tmp_dir.path().join(&name);
     let file = File::create(&file_path).await?;
     let mut a = tokio_tar::Builder::new(file);
+     
 
     {
         let scripts = sqlx::query_as::<_, Script>(
-            "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
-            AND created_at = (select max(created_at) from script where path = o.path AND \
-             workspace_id = $1)",
-        )
-        .bind(&w_id)
-        .fetch_all(&db)
-        .await?;
+        "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
+            AND created_at = (select max(created_at) from script where path = o.path AND workspace_id = $1)"
+    )
+    .bind(&w_id)
+    .fetch_all(&db)
+    .await?;
+
+    for script in scripts {
+        write_to_archive(script.content, format!("scripts/{}.py", script.path), &mut a).await?;
+
+        let lock = script.lock.unwrap_or_else(|| "".to_string())
+        .lines()
+        .map(|x| x.to_string())
+        .collect();
+        let metadata = ScriptMetadata { 
+            summary: script.summary, description: script.description, schema: script.schema, is_template: script.is_template, lock };
+        let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
+        write_to_archive(metadata_str, format!("scripts/{}.json", script.path), &mut a).await?;
+
+    };
+}
+
+
 
-        for script in scripts {
-            write_to_archive(
-                script.content,
-                format!("scripts/{}.py", script.path),
-                &mut a,
-            )
-            .await?;
 
-            let lock = script
-                .lock
-                .unwrap_or_else(|| "".to_string())
-                .lines()
-                .map(|x| x.to_string())
-                .collect();
-            let metadata = ScriptMetadata {
-                summary: script.summary,
-                description: script.description,
-                schema: script.schema,
-                is_template: script.is_template,
-                lock,
-            };
-            let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
-            write_to_archive(
-                metadata_str,
-                format!("scripts/{}.json", script.path),
-                &mut a,
-            )
-            .await?;
-        }
-    }
 
     {
-        let resources = sqlx::query_as!(
-            Resource,
+
+        let resources = sqlx::query_as!(Resource,
             "SELECT * FROM resource WHERE workspace_id = $1",
             &w_id
         )
@@ -595,38 +589,27 @@ async fn tarball_workspace(
 
         for resource in resources {
             let resource_str = serde_json::to_string_pretty(&resource).unwrap();
-            write_to_archive(
-                resource_str,
-                format!("resources/{}.json", resource.path),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(resource_str, format!("resources/{}.json", resource.path), &mut a).await?;
         }
     }
 
     {
-        let resource_types = sqlx::query_as!(
-            ResourceType,
-            "SELECT * FROM resource_type WHERE workspace_id = $1",
-            &w_id
-        )
-        .fetch_all(&db)
-        .await?;
+        let resource_types = sqlx::query_as!(ResourceType,
+        "SELECT * FROM resource_type WHERE workspace_id = $1",
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
 
         for resource_type in resource_types {
             let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
-            write_to_archive(
-                resource_str,
-                format!("resource_types/{}.json", resource_type.name),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(resource_str, format!("resource_types/{}.json", resource_type.name), &mut a).await?;
         }
     }
 
     {
         let flows = sqlx::query_as::<_, Flow>(
-            "SELECT * FROM flow WHERE workspace_id = $1 AND archived = false",
+            "SELECT * FROM flow WHERE workspace_id = $1 AND archived = false"
         )
         .bind(&w_id)
         .fetch_all(&db)
@@ -639,8 +622,8 @@ async fn tarball_workspace(
     }
 
     {
-        let variables = sqlx::query_as::<_, ListableVariable>(
-            "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1 AND is_secret = false",
+       let variables = sqlx::query_as::<_, ListableVariable>(
+        "SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false"
         )
         .bind(&w_id)
         .fetch_all(&db)
@@ -669,11 +652,7 @@ async fn tarball_workspace(
     Ok((headers, body))
 }
 
-async fn write_to_archive(
-    content: String,
-    path: String,
-    a: &mut tokio_tar::Builder<File>,
-) -> Result<()> {
+async fn write_to_archive(content: String, path: String, a: &mut tokio_tar::Builder<File>) -> Result<()> {
     let bytes = content.as_bytes();
     let mut header = tokio_tar::Header::new_gnu();
     header.set_size(bytes.len() as u64);