chore(main): release 1.27.2 (#322)

* chore(main): release 1.27.2

* Apply automatic changes

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -1,3 +1 @@
-SITE_URL=localhost
 DB_PASSWORD=changeme
-POSTGRES_VERSION=13.3.0

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @rubenfiszel

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [rubenfiszel]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +1,27 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
+title: 'bug:'
+labels: 'bug'
+assignees: 'rubenfiszel'
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
+**Describe the bug** A clear and concise description of what the bug is.
+
+**To Reproduce** Steps to reproduce the behavior:
 
-**To Reproduce**
-Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior**
-A clear and concise description of what you expected to happen.
+**Expected behavior** A clear and concise description of what you expected to
+happen.
 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
+**Screenshots** If applicable, add screenshots to help explain your problem.
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+**Windmill version** Go on the left menu -> <user> -> User Settings and copy the
+printed version in "Running windmill version (backend): XXX".
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.
+**Additional context** Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,8 @@
+---
+name: Feature Request
+about: Create a feature request
+title: 'feature: '
+labels: 'feature'
+assignees: 'rubenfiszel'
+
+---

.github/change-versions.sh

@@ -5,12 +5,15 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml
+sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill/pyproject.toml
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" Pipfile
-sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" Pipfile
+# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
+sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
+sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
 
 sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" backend/Cargo.lock
+
+cd frontend && npm i --package-lock-only

.github/dependabot.yml

@@ -0,0 +1,39 @@
+# Basic set up for three package managers
+
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for npm
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for cargo
+  - package-ecosystem: "cargo"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for Docker
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for wmill python client
+  - package-ecosystem: "pip"
+    directory: "/python-client/wmill"
+    schedule:
+      interval: "weekly"
+      
+  # Maintain dependencies for wmill_pg python client
+  - package-ecosystem: "pip"
+    directory: "/python-client/wmill_pg"
+    schedule:
+      interval: "weekly"

.github/pull_hub_items.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+RT=$(curl -s https://hub.windmill.dev/resource_types/list | jq -c -r '.[]')
+for item in ${RT[@]}; do
+    name=$(jq -r '.name' <<< "$item")
+    id=$(jq -r '.id' <<< "$item")
+    echo $name $id 
+    body=$(curl -s -H "accept: application/json" https://hub.windmill.dev/resource_types/${id}/${name})
+    jq -r '.resource_type.schema' <<< "$body" > ./tmp
+    description=$(jq -r '.resource_type.description' <<< "$body")
+    echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
+    rm ./tmp
+done

.github/workflows/automerge-dependabot.yml

@@ -0,0 +1,26 @@
+name: dependabot auto-merge
+
+on: pull_request_target
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.3.3
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Enable auto-merge for Dependabot PRs
+        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
+        run: |
+          echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
+          gh pr review --approve "$PR_URL"
+          gh pr merge --auto --squash "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/change-versions.yml

@@ -7,8 +7,9 @@ on:
 jobs:
   change_version:
     runs-on: ubuntu-latest
+    container: node:18
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
       - uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/deno_on_release.yml

@@ -0,0 +1,49 @@
+name: Publish deno-client
+on:
+  push:
+    tags:
+      - "v*"
+env:
+  repo: windmill-deno-client
+
+jobs:
+  build_deno_and_push_to_repo:
+    runs-on: ubuntu-latest
+    container: openapitools/openapi-generator-cli:v6.0.0-beta
+    steps:
+      - uses: actions/checkout@v3
+      - name: generate_deno
+        run: |
+          cd deno-client
+          rm .gitignore
+          ./generate.sh
+      - name: Pushes to another repository
+        id: push_directory
+        uses: cpina/github-action-push-to-another-repository@devel
+        env:
+          API_TOKEN_GITHUB: ${{ secrets.DENO_PAT }}
+        with:
+          source-directory: deno-client/
+          destination-github-username: ${{ github.repository_owner }}
+          destination-repository-name: ${{ env.repo }}
+          user-email: ruben@windmill.dev
+          commit-message: See ORIGIN_COMMIT from $GITHUB_REF
+          target-branch: main
+
+  tag_repo:
+    needs: [build_deno_and_push_to_repo]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository_owner }}/${{ env.repo }}
+          token: ${{ secrets.DENO_PAT }}
+          path: ./client
+
+      - name: Push client
+        run: |
+          cd ./client
+          git config --global user.email "ruben@windmill.dev"
+          git config --global user.name "rubenfiszel[bot]"
+          git tag -a ${{ github.ref_name }} -m "${{ github.ref_name }}"
+          git push --tags

.github/workflows/deploy_to_windmill.yml

@@ -3,6 +3,8 @@ name: Deploy to windmill.dev
 on:
   push:
     branches: [main]
+    paths:
+      - "community/**"
 
 jobs:
   deploy:
@@ -10,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v1.0.0
+        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
         with:
           dry_run: false
           input_dir: community

.github/workflows/docker-image.yml

@@ -1,7 +1,13 @@
-name: Docker Image CI
+env:
+  LOCAL_REGISTRY: registry.wimill.xyz
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build and push docker image
 on:
   push:
     branches: [main]
+    tags: ["*"]
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -12,28 +18,95 @@ concurrency:
 jobs:
   build:
     runs-on: [self-hosted, new]
-    env:
-      DOCKER_BUILDKIT: 1
     steps:
-      - name: Wait for release to succeed
-        if: github.ref == 'refs/heads/main'
-        uses: lewagon/wait-on-check-action@v1.0.0
-        with:
-          ref: ${{ github.ref }}
-          check-name: "Release please"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 10
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: deploy staging stack
-        run: |
-          docker build . --cache-from "registry.wimill.xyz/windmill:staging" -t "registry.wimill.xyz/windmill:staging" --build-arg BUILDKIT_INLINE_CACHE=1
-          docker push "registry.wimill.xyz/windmill:staging"
-      - name: deploy demo stack
-        if: github.ref == 'refs/heads/main'
-        run: |
-          docker tag registry.wimill.xyz/windmill:staging registry.wimill.xyz/windmill:main
-          docker push registry.wimill.xyz/windmill:main
-      # - name: pruning unused images
-      #   run: sudo docker image prune -a
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker meta local
+        id: metalocal
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push privately
+        uses: docker/build-push-action@v3
+        if: github.event_name == 'pull_request'
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+
+      - name: Docker meta
+        if: github.event_name != 'pull_request'
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Login to registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly
+        uses: docker/build-push-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+            ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+  playwright:
+    runs-on: [self-hosted, new]
+    needs: [build]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_USER: admin
+          POSTGRES_PASSWORD: changeme
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Docker"
+        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+        id: docker-container
+      - name: "Playwright run"
+        timeout-minutes: 10
+        run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
+      - name: "Clean up"
+        run: docker kill ${{ steps.docker-container.outputs.id }}
+        if: always()

.github/workflows/on-release.yml

@@ -1,38 +0,0 @@
-name: Build LSP Docker
-on:
-  push:
-    branches: [main]
-    paths:
-      - "python-client/**"
-      - "Pipfile"
-      - ".github/workflows/on-release.yml"
-
-jobs:
-  build_lsp:
-    runs-on: [self-hosted, new]
-    steps:
-      - name: Wait for release to succeed
-        if: github.ref == 'refs/heads/main'
-        uses: lewagon/wait-on-check-action@v1.0.0
-        with:
-          ref: ${{ github.ref }}
-          check-name: "Release please"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 10
-      - uses: actions/checkout@v2
-      - name: Upload python client
-        env:
-          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          cd python-client
-          export PATH=$PATH:/usr/local/bin
-          export PATH=$PATH:/root/.local/bin
-          ./publish.sh
-      - name: Build the Docker image
-        run: |
-          cd lsp
-          sudo docker pull "registry.wimill.xyz/lsp:main" || true
-          sudo docker build  .  --cache-from "registry.wimill.xyz/lsp:main" -t "registry.wimill.xyz/lsp:main" --build-arg BUILDKIT_INLINE_CACHE=1
-      - name: push to registry
-        run: |
-          sudo docker push "registry.wimill.xyz/lsp:main"

.github/workflows/pull-hub.yml

@@ -0,0 +1,19 @@
+name: Pull Hub Items
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 0 */1 * *"
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  change_version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Pull hub
+        run: ./.github/pull_hub_items.sh
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          title: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -0,0 +1,52 @@
+env:
+  LOCAL_REGISTRY: registry.wimill.xyz
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Publish python-client
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish_pypi:
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Upload python client
+        env:
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          cd python-client
+          export PATH=$PATH:/usr/local/bin
+          export PATH=$PATH:/root/.local/bin
+          ./publish.sh
+
+  publish_lsp:
+    needs: [publish_pypi]
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Docker meta local
+        id: metalocal
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:lsp"
+          push: true
+          tags: ${{ steps.metalocal.outputs.tags }}
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp:buildcache,mode=max

.github/workflows/release-please.yml

@@ -1,14 +1,14 @@
 on:
   push:
-    branches:
-      - main
+    branches: [main]
+
 name: release-please
 jobs:
   release-please:
     name: "Release please"
     runs-on: ubuntu-latest
     steps:
-      - uses: GoogleCloudPlatform/release-please-action@v2
+      - uses: GoogleCloudPlatform/release-please-action@v3
         with:
           release-type: simple
           package-name: windmill

.github/workflows/sign-cla.yml

@@ -0,0 +1,34 @@
+name: "CLA Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened, closed, synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "CLA Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        # Beta Release
+        uses: cla-assistant/github-action@v2.2.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}
+        with:
+          path-to-signatures: "signatures/cla.json"
+          path-to-document: "https://github.com/windmill-labs/windmill/blob/master/CLA.md"
+          branch: "signatures"
+          allowlist: rubenfiszel,bot*
+
+          #below are the optional inputs - If the optional inputs are not given, then default values will be taken
+          #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #remote-repository-name:  enter the  remote repository name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #create-file-commit-message: 'For example: Creating file for storing CLA Signatures'
+          #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo'
+          #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign'
+          #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA'
+          #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.'
+          #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true)
+          #use-dco-flag: true - If you are using DCO instead of CLA

CHANGELOG.md

@@ -1,3 +1,524 @@
 # Changelog
 
+
+## [1.27.2](https://github.com/windmill-labs/windmill/compare/v1.27.1...v1.27.2) (2022-08-02)
+
+
+### Bug Fixes
+
+* **deno-client:** getResource can now fetch non-object values ([b128388](https://github.com/windmill-labs/windmill/commit/b128388cc652d4cd369a88b93985a2c051003abd))
+
+## [1.27.1](https://github.com/windmill-labs/windmill/compare/v1.27.0...v1.27.1) (2022-08-02)
+
+
+### Bug Fixes
+
+* migrate to new style radio button ([893ee94](https://github.com/windmill-labs/windmill/commit/893ee941d72a7036f0ea272c49bbe5cd3eee64d5))
+
+## [1.27.0](https://github.com/windmill-labs/windmill/compare/v1.26.3...v1.27.0) (2022-08-02)
+
+
+### Features
+
+* add primitive sql format ([#320](https://github.com/windmill-labs/windmill/issues/320)) ([9daff2a](https://github.com/windmill-labs/windmill/commit/9daff2a228791234a3dd70c0ee829e284daf1592))
+
+
+### Bug Fixes
+
+* prefer `COPY` over `ADD` ([#319](https://github.com/windmill-labs/windmill/issues/319)) ([24a7e46](https://github.com/windmill-labs/windmill/commit/24a7e46fe99d5a1f7d5b22334fa5f6ce76e82d94))
+* typos ([#301](https://github.com/windmill-labs/windmill/issues/301)) ([9e84b45](https://github.com/windmill-labs/windmill/commit/9e84b458b139e86eb51dba9c5b228f141ca649b3))
+
+## [1.26.3](https://github.com/windmill-labs/windmill/compare/v1.26.2...v1.26.3) (2022-08-01)
+
+
+### Bug Fixes
+
+* displaying which group you are a member of that gave you access to item ([1bd0269](https://github.com/windmill-labs/windmill/commit/1bd026924b8a3b01f7729b627f939d8af872a483))
+* refresh jobs result when hopping from flow to flow ([c86abe6](https://github.com/windmill-labs/windmill/commit/c86abe6ae01efd519f67ead233ebddf39f1539c0))
+
+## [1.26.2](https://github.com/windmill-labs/windmill/compare/v1.26.1...v1.26.2) (2022-07-31)
+
+
+### Bug Fixes
+
+* deno api generator now supports openflow ([5b548a0](https://github.com/windmill-labs/windmill/commit/5b548a0e71669aad90343e70f3f1c9dc3a6d4baf))
+
+## [1.26.1](https://github.com/windmill-labs/windmill/compare/v1.26.0...v1.26.1) (2022-07-31)
+
+
+### Bug Fixes
+
+* encoding state now supports unicode including emojis ([6b61227](https://github.com/windmill-labs/windmill/commit/6b61227481422fe52384f6de8146388a8471ff60))
+
+## [1.26.0](https://github.com/windmill-labs/windmill/compare/v1.25.0...v1.26.0) (2022-07-29)
+
+
+### Features
+
+* resource type picker in schema modal + proper initialization of raw javascript editor when applicable ([01bb107](https://github.com/windmill-labs/windmill/commit/01bb107a0f3e3899ec99718974b2484ab5978c92))
+
+
+### Bug Fixes
+
+* forloop flows unsoundness fix part I ([1b5ce32](https://github.com/windmill-labs/windmill/commit/1b5ce3243b364d02903072a9af5e15447622e9fb))
+* small bar mode and editor nits ([4e3a02a](https://github.com/windmill-labs/windmill/commit/4e3a02a8e44e25e6b5402f732b9af6969d06dcc0))
+
+## [1.25.0](https://github.com/windmill-labs/windmill/compare/v1.24.2...v1.25.0) (2022-07-29)
+
+
+### Features
+
+* base64 support in schema editor ([2cb6e6e](https://github.com/windmill-labs/windmill/commit/2cb6e6e7021819a9aa9618436abf2f0fa5b3587b))
+
+
+### Bug Fixes
+
+* update variable and resources now return error if nothing was updated ([0faabdb](https://github.com/windmill-labs/windmill/commit/0faabdbc40b049258b074c6c20c1406ca14b8481))
+
+## [1.24.2](https://github.com/windmill-labs/windmill/compare/v1.24.1...v1.24.2) (2022-07-28)
+
+
+### Bug Fixes
+
+* get_variable refresh_token bug ([390e9b3](https://github.com/windmill-labs/windmill/commit/390e9b37fb201242ac6983c145c9de5b242f7a7b))
+* if :path is not a valid path, do not even attempt to fetch it ([6dec447](https://github.com/windmill-labs/windmill/commit/6dec4479537164fe17bea7f88fd60b1d4f42e887))
+* monaco editor fixes ([f255cc2](https://github.com/windmill-labs/windmill/commit/f255cc253fcf14850442e8d4bf64635287b88314))
+
+## [1.24.1](https://github.com/windmill-labs/windmill/compare/v1.24.0...v1.24.1) (2022-07-27)
+
+
+### Bug Fixes
+
+* encrypt the refresh token ([a051c21](https://github.com/windmill-labs/windmill/commit/a051c2121a63983f6925ce2e3a9b9deb01df2f04))
+* keep previous refresh token if no new ones were provided ([3feef73](https://github.com/windmill-labs/windmill/commit/3feef738dc145603576649a91f0ddc0e82215841))
+* skip_failures is boolean not bool ([4ca71c1](https://github.com/windmill-labs/windmill/commit/4ca71c1e5da0132724ab4c9771f5fdc590b866f8))
+
+## [1.24.0](https://github.com/windmill-labs/windmill/compare/v1.23.0...v1.24.0) (2022-07-27)
+
+
+### Features
+
+* Add flow input and current step in the prop picker ([#236](https://github.com/windmill-labs/windmill/issues/236)) ([6fbeeae](https://github.com/windmill-labs/windmill/commit/6fbeeae84a207be46490361788dad12918c37c4e))
+* add google login v1 ([fc918a2](https://github.com/windmill-labs/windmill/commit/fc918a24ccf0ad19b81a3ebf630d0f04b56094c8))
+* add schedule settable from pull flows ([caecbfd](https://github.com/windmill-labs/windmill/commit/caecbfd0d9eaadc38372ce7238ed6d3baf9ba6e3))
+* prop picker functional for pull flows ([010acfe](https://github.com/windmill-labs/windmill/commit/010acfe7e365a838078f1a989b54f1539c8bf2e6))
+* skip failures loop ([#258](https://github.com/windmill-labs/windmill/issues/258)) ([de3fe69](https://github.com/windmill-labs/windmill/commit/de3fe699089e2a28aa0032a57a9a03f35646b6ef))
+
+
+### Bug Fixes
+
+* audit logs ([ca4bed3](https://github.com/windmill-labs/windmill/commit/ca4bed34a65440cd790cae9cff19f40df22f92b8))
+* **frontend:** badge google logo for login ([cfec7a9](https://github.com/windmill-labs/windmill/commit/cfec7a97b883dbf83bd9d0707bf015c2aaa4e517))
+* **frontend:** badge needs a little right margin ([c846ed7](https://github.com/windmill-labs/windmill/commit/c846ed76c4102335a5a8aabceaa39d6b7906ef5a))
+* **frontend:** display number field in flows ([a232895](https://github.com/windmill-labs/windmill/commit/a23289563deca70269bd73ec50f324db0b6df791))
+* **frontend:** fork script from hub ([43cacc1](https://github.com/windmill-labs/windmill/commit/43cacc1a66b1e2322c0252c9d1ca954e893aaef8))
+* **frontend:** get refresh token for google services ([2f0d8d5](https://github.com/windmill-labs/windmill/commit/2f0d8d5384fb4eea6a6d5e5e48fd242f8d0c40fa))
+* **frontend:** get refresh token for google services ([8dfe688](https://github.com/windmill-labs/windmill/commit/8dfe688a6a2388cecb1460913a25ab49ec297b1b))
+* **frontend:** get refresh token for google services ([a2c5dc1](https://github.com/windmill-labs/windmill/commit/a2c5dc18a38045cbefc7d4b86d786a3c8fcb3ca8))
+* import from JSON load schemas ([88dd7b0](https://github.com/windmill-labs/windmill/commit/88dd7b0abbd1a0469fc949c8045f61ddc304701d))
+* multiple UI fixes ([a334029](https://github.com/windmill-labs/windmill/commit/a33402978720470530baecf51c2d17ecafd13ab0))
+* multiple UI fixes ([904f0f3](https://github.com/windmill-labs/windmill/commit/904f0f3e69034421d524a66e0c4697ff42d89efe))
+
+## [1.23.0](https://github.com/windmill-labs/windmill/compare/v1.22.0...v1.23.0) (2022-07-25)
+
+
+### Features
+
+* add editor bar to inline scripts of flows ([7a6a2c9](https://github.com/windmill-labs/windmill/commit/7a6a2c982daef9aa80e34aa6cbd4889a3c5ec807))
+* **backend:** do not require visibility on job to see job if in possesion of uuid ([b054229](https://github.com/windmill-labs/windmill/commit/b05422963b27d74de8bb6d3be18538d57a71cfe7))
+* **frontend:** deeper integration with the hub ([bb58eba](https://github.com/windmill-labs/windmill/commit/bb58eba2b521aef67b91cfc23f3ddcc8a001e18f))
+* **frontend:** title everywhere ([38987c6](https://github.com/windmill-labs/windmill/commit/38987c6068c4cc2d9accbc368a67362e74adcabf))
+* hub flows integration ([62777b7](https://github.com/windmill-labs/windmill/commit/62777b7a7888b3456f7f864cbb1acd887b172adc))
+
+
+### Bug Fixes
+
+* display websocket status in flow inline editor ([9e9138e](https://github.com/windmill-labs/windmill/commit/9e9138e4eeaea962dbb149ad4c1450572f025bc5))
+* do not redirect to /user on /user namespace ([d95128e](https://github.com/windmill-labs/windmill/commit/d95128e68190fa6f75871f579de906ce82619524))
+* **oauth2:** add google clients ([bc650b0](https://github.com/windmill-labs/windmill/commit/bc650b0ade1d378f815ee01da480a63ddd4501f1))
+* static is undefined by default instead of being empty '' ([fc65162](https://github.com/windmill-labs/windmill/commit/fc651629c7977b5221dbb101f515766b23af9274))
+
+## [1.22.0](https://github.com/windmill-labs/windmill/compare/v1.21.1...v1.22.0) (2022-07-22)
+
+
+### Features
+
+* add delete schedule ([f6d6934](https://github.com/windmill-labs/windmill/commit/f6d69345841f2ec0d06dc32b59840009982c55f2))
+* **backend:** check of no path conflict between flow and flow's primary schedules ([c346339](https://github.com/windmill-labs/windmill/commit/c34633989e41e215d6183e5c887db68d4cc228d3))
+* dynamic template for script inputs in flow ([3c16621](https://github.com/windmill-labs/windmill/commit/3c16621f6b9c2bee1f2630411bd70d075d247974))
+* import and export flow from JSON ([7862ff4](https://github.com/windmill-labs/windmill/commit/7862ff41e25447d7b34aa261187bb98ed3f3105b))
+* more visual cues about trigger scripts ([36606ab](https://github.com/windmill-labs/windmill/commit/36606ab8b675d01b0d38e2dd883b6e42b0987a6c))
+* more visual cues about trigger scripts ([154c2a9](https://github.com/windmill-labs/windmill/commit/154c2a91ca6a4d60b02a44dda5fa23974594018b))
+* rich rendering of flows ([38ffcfe](https://github.com/windmill-labs/windmill/commit/38ffcfeb292c6e9df0c89a4ef5364cdb8e23ccdd))
+
+
+### Bug Fixes
+
+* **deno-client:** make hack for patching openapi-generator more stable ([08ab4d1](https://github.com/windmill-labs/windmill/commit/08ab4d171a286d94e439a89d97115ad2db8e25d9))
+* export json is converted to pull mode ([666e0f6](https://github.com/windmill-labs/windmill/commit/666e0f68d0dd84fce35e6fe1804c90a3c5125057))
+* export json is converted to pull mode + rd fix ([c7528d4](https://github.com/windmill-labs/windmill/commit/c7528d417f276fbdb96751cda547feec7ac6fbc8))
+* **frontend:** filter script by is_trigger and jobs by is_skipped + path fix ([97292d1](https://github.com/windmill-labs/windmill/commit/97292d18fb7158471f1be6ffbd45a612b09a689f))
+* **frontend:** initFlow also reset schemaStore ([5941467](https://github.com/windmill-labs/windmill/commit/5941467ea19938b4d11b56c6f10f529c87cb52a3))
+* **frontend:** remove unecessary step 1 of flows ([f429074](https://github.com/windmill-labs/windmill/commit/f429074528770f5eaebcf1ce687b6431321e169a))
+* improve tooltip ([4be5d37](https://github.com/windmill-labs/windmill/commit/4be5d37a5441555c83eefbea17e86a5df4946749))
+* improve tooltip ([c84b1c9](https://github.com/windmill-labs/windmill/commit/c84b1c9a8c6a03b9689e3405fa87f3c54016914a))
+* placeholder undefined for arginput ([4d01598](https://github.com/windmill-labs/windmill/commit/4d01598e24fca673b0dc83860e151c21ab403b7a))
+
+## [1.21.1](https://github.com/windmill-labs/windmill/compare/v1.21.0...v1.21.1) (2022-07-19)
+
+
+### Bug Fixes
+
+* **deno-client:** make hack for patching openapi-generator more stable ([2f4df43](https://github.com/windmill-labs/windmill/commit/2f4df43a1a798501449e82767d59f08e9cf95146))
+* **python-client:** sed openapi to avoid generator circular dependency ([49f8050](https://github.com/windmill-labs/windmill/commit/49f8050aaf48c15fb79130a06ce754e285d17dd0))
+
+## [1.21.0](https://github.com/windmill-labs/windmill/compare/v1.20.0...v1.21.0) (2022-07-19)
+
+
+### Features
+
+* add run_wait_result to mimic lambda ability ([6ef3754](https://github.com/windmill-labs/windmill/commit/6ef3754759346b8261934a35bd3bf3983872390f))
+
+
+### Bug Fixes
+
+* **backend:** clear env variables before running script ([98a5959](https://github.com/windmill-labs/windmill/commit/98a5959fcca19c54715e78055cf8881496209ac0))
+* consistent exists/{resource} addition + usage in frontend ([ca66d33](https://github.com/windmill-labs/windmill/commit/ca66d33a4297d2f3a105829650a544f4a89c4615))
+* **frontend:** validate username ([9828e54](https://github.com/windmill-labs/windmill/commit/9828e545e9649bc2ac6af598118ef85580fd80f3))
+* list with is_skipped + deno-client fix ([6939f9d](https://github.com/windmill-labs/windmill/commit/6939f9d76b1579f2932e08df3f67dc293c642fd0))
+
+## [1.20.0](https://github.com/windmill-labs/windmill/compare/v1.19.3...v1.20.0) (2022-07-17)
+
+
+### Features
+
+* trigger scripts and have flows being triggered by checking new external events regularly ([#200](https://github.com/windmill-labs/windmill/issues/200)) ([af23b30](https://github.com/windmill-labs/windmill/commit/af23b30c37b4225d6b927644f9612d4861e2d06c))
+
+
+### Bug Fixes
+
+* flow UI back and forth pull/push fix ([8918eb6](https://github.com/windmill-labs/windmill/commit/8918eb6fdb904e23b5dc340db669f6039ed7abb6))
+* flow UI back and forth pull/push fix ([0973859](https://github.com/windmill-labs/windmill/commit/097385981323d5f88a51eb8df0e1114e8cf62727))
+* **frontend:** chrome columns-2 fix for pull/push ([8272b11](https://github.com/windmill-labs/windmill/commit/8272b1110757ee0ed0cee4a7a6de537fcec83de3))
+* **frontend:** createInlineScript only create trigger script if step = 0 ([bd004cf](https://github.com/windmill-labs/windmill/commit/bd004cff0f5150eb043f5446f5697bea43b1508b))
+* HubPicker pick from trigger scripts when relevant ([7e846c3](https://github.com/windmill-labs/windmill/commit/7e846c32a63d9fe2f46f50f7642918cc34459829))
+
+## [1.19.3](https://github.com/windmill-labs/windmill/compare/v1.19.2...v1.19.3) (2022-07-15)
+
+
+### Bug Fixes
+
+* **deno-client:** do not create resource for createInternalPath ([0967c1b](https://github.com/windmill-labs/windmill/commit/0967c1be65a9803e25f7701850be33121eb44d1b))
+
+## [1.19.2](https://github.com/windmill-labs/windmill/compare/v1.19.1...v1.19.2) (2022-07-15)
+
+
+### Bug Fixes
+
+* **deno-client:** handle text/plain parse ([18e33bb](https://github.com/windmill-labs/windmill/commit/18e33bb40739fd699323f2da87de8c9696c0ef6c))
+
+## [1.19.1](https://github.com/windmill-labs/windmill/compare/v1.19.0...v1.19.1) (2022-07-14)
+
+
+### Bug Fixes
+
+* **backend:** create resource would fail if is_oauth was not set ([cd621a6](https://github.com/windmill-labs/windmill/commit/cd621a6285d2aa0e554434998e931e96110464bd))
+* **deno-client:** handle text/plain serialize ([98968ab](https://github.com/windmill-labs/windmill/commit/98968ab039fea89b7525fe7b852ba3d15dee831e))
+
+## [1.19.0](https://github.com/windmill-labs/windmill/compare/v1.18.0...v1.19.0) (2022-07-14)
+
+
+### Features
+
+* add DISABLE_NSJAIL mode ([1943585](https://github.com/windmill-labs/windmill/commit/19435851de0c18fc876a3bd00f3d9153f2719d9b))
+
+
+### Bug Fixes
+
+* add new ca-certificates folders for nsjail ([2eac1ef](https://github.com/windmill-labs/windmill/commit/2eac1ef363b209bb298dcbe7aafb7282ddd2b87a))
+* **frontend:** add arbitrary scopes to connect an app ([372b14e](https://github.com/windmill-labs/windmill/commit/372b14e158bcb10bcfb07d231afeca5cc780661d))
+* write job arguments to file ([#199](https://github.com/windmill-labs/windmill/issues/199)) ([9a6db75](https://github.com/windmill-labs/windmill/commit/9a6db758c15915f5f0027b1d270d621f91b7ae30))
+
+## [1.18.0](https://github.com/windmill-labs/windmill/compare/v1.17.1...v1.18.0) (2022-07-13)
+
+
+### Features
+
+* account part II, handle refresh tokens, clarify oauth UI ([#196](https://github.com/windmill-labs/windmill/issues/196)) ([8403fbb](https://github.com/windmill-labs/windmill/commit/8403fbbc02076bb37dc82b2d26685957b13d036b))
+
+
+### Bug Fixes
+
+* **frontend:** fix path group refresh & create variable path reset ([6a341f5](https://github.com/windmill-labs/windmill/commit/6a341f5dc343df3df6491f8026e87632979faace))
+
+## [1.17.1](https://github.com/windmill-labs/windmill/compare/v1.17.0...v1.17.1) (2022-07-08)
+
+
+### Bug Fixes
+
+* **backend:** set error content-type to text ([cf2dfd7](https://github.com/windmill-labs/windmill/commit/cf2dfd7fe74956d68bdc26dc47557ea6a0ed1ce4))
+* **deno-client:** fix stringify ([5b89abe](https://github.com/windmill-labs/windmill/commit/5b89abe28283238a282da8920580a72f25e5a360))
+* **frontend:** change lsp behavior ([d6e0817](https://github.com/windmill-labs/windmill/commit/d6e0817dc4fe54efd9346698c0ccb39057921d9b))
+* **frontend:** connect an app resource creation ([e400dcc](https://github.com/windmill-labs/windmill/commit/e400dccedd88e3f5e3a9b0ec52fc9883d60c959b))
+* **frontend:** connect an app resource creation ([68c5318](https://github.com/windmill-labs/windmill/commit/68c5318d16c85a01822570c113a4f33c539dc8bf))
+* **frontend:** current hash link ([22eef8a](https://github.com/windmill-labs/windmill/commit/22eef8afab9143bb5b110db8c76e024604106051))
+* **frontend:** fix sendRequest ([5da9819](https://github.com/windmill-labs/windmill/commit/5da9819ca5ce15ef4de9cf4a84affbd581383483))
+* **frontend:** reload editor when language changes for in-flow editor ([72c7890](https://github.com/windmill-labs/windmill/commit/72c7890427736eeeb9a872bf0efd1acc906efd63))
+* **frontend:** sveltekit prerender enabled -> default ([635873a](https://github.com/windmill-labs/windmill/commit/635873a96a586ad8e936526f4f4ebf679519e7fc))
+* in-flow script editor fixes ([466f6b3](https://github.com/windmill-labs/windmill/commit/466f6b339acf70351814c32b8f31d80b8ff1c1b5))
+* in-flow script editor fixes ([5853dfd](https://github.com/windmill-labs/windmill/commit/5853dfd85dca3c80b0edfb58b2866948af8011d5))
+* remove unnecessary v8 snapshot ([d3904fd](https://github.com/windmill-labs/windmill/commit/d3904fd3ebde3a200ccc157a8532dfe1435ae16d))
+
+## [1.17.0](https://github.com/windmill-labs/windmill/compare/v1.16.1...v1.17.0) (2022-07-05)
+
+
+### Features
+
+* in-flow editor mvp ([330b373](https://github.com/windmill-labs/windmill/commit/330b373c24f21b4d9a9b2903e8f1c60ee784ea89))
+
+## [1.16.1](https://github.com/windmill-labs/windmill/compare/v1.16.0...v1.16.1) (2022-07-05)
+
+
+### Bug Fixes
+
+* bump all backend deps by breaking cycling through not using oauth2 ([e4a6378](https://github.com/windmill-labs/windmill/commit/e4a637860133e78cb1675173ccf3ff45e4b08c09))
+* oauth logins used incorrect scope ([1dcba67](https://github.com/windmill-labs/windmill/commit/1dcba67a1f607faabcdfa6f7e94d280c66dd6470))
+* trace errors body ([d092c62](https://github.com/windmill-labs/windmill/commit/d092c622c4efadb1e2799f7dbbe03f825f2b364d))
+
+
+## [1.16.0](https://github.com/windmill-labs/windmill/compare/v1.15.1...v1.16.0) (2022-07-02)
+
+
+### Features
+
+* OAuth "Connect an App" ([#155](https://github.com/windmill-labs/windmill/issues/155)) ([3636866](https://github.com/windmill-labs/windmill/commit/3636866dda8b2e14d61c99a76f0a4e5fa6a37123))
+
+
+### Bug Fixes
+
+* add gitlab to connects ([d4e7c9e](https://github.com/windmill-labs/windmill/commit/d4e7c9e171cd02a7aa0846b43c127720260600b5))
+* diverse frontend fixes
+
+## [1.15.1](https://github.com/windmill-labs/windmill/compare/v1.15.0...v1.15.1) (2022-06-29)
+
+
+### Bug Fixes
+
+* databaseUrlFromResource uses proper database field ([6954580](https://github.com/windmill-labs/windmill/commit/69545808012fa4f5080ec58cf3dff2961a327117))
+
+## [1.15.0](https://github.com/windmill-labs/windmill/compare/v1.14.6...v1.15.0) (2022-06-29)
+
+
+### Features
+
+* Flows Property picker component + Dynamic type inference ([#129](https://github.com/windmill-labs/windmill/issues/129)) ([44b4acf](https://github.com/windmill-labs/windmill/commit/44b4acf4bcfa0c372a9938a9b97d31cceedd9ad9))
+
+## [1.14.6](https://github.com/windmill-labs/windmill/compare/v1.14.5...v1.14.6) (2022-06-27)
+
+
+### Bug Fixes
+
+* add databaseUrlFromResource to deno ([2659e9d](https://github.com/windmill-labs/windmill/commit/2659e9d62b88c2127c969becbc3a61ed2f118069))
+
+## [1.14.5](https://github.com/windmill-labs/windmill/compare/v1.14.4...v1.14.5) (2022-06-27)
+
+
+### Bug Fixes
+
+* index.ts -> mod.ts ([d41913a](https://github.com/windmill-labs/windmill/commit/d41913a440b2034de59437488edc85e38c956d5f))
+* insert getResource proper parenthesis ([e07b5d4](https://github.com/windmill-labs/windmill/commit/e07b5d4f30ea79a99caac4fb63a9ab1f17eaaf74))
+
+## [1.14.4](https://github.com/windmill-labs/windmill/compare/v1.14.3...v1.14.4) (2022-06-27)
+
+
+### Bug Fixes
+
+* windmill deno package index.ts -> mod.ts ([8c0acac](https://github.com/windmill-labs/windmill/commit/8c0acac212d742acee8b7ff0cf6b93cce4187c19))
+
+## [1.14.3](https://github.com/windmill-labs/windmill/compare/v1.14.2...v1.14.3) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v3 ([31445d7](https://github.com/windmill-labs/windmill/commit/31445d7182a910eab9d699760f2a86ca23d556a4))
+* internal state for script triggers v3 ([22c6347](https://github.com/windmill-labs/windmill/commit/22c6347d8a74d94dc18109390ff5c347a2732823))
+* internal state for script triggers v4 ([63a7401](https://github.com/windmill-labs/windmill/commit/63a7401f248cc37951bbea4dcaedaa6497d6f0b1))
+
+## [1.14.2](https://github.com/windmill-labs/windmill/compare/v1.14.1...v1.14.2) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v2 ([f9eedc3](https://github.com/windmill-labs/windmill/commit/f9eedc31ed6e5d7e0a8a26633cca9965ac3b6a05))
+
+## [1.14.1](https://github.com/windmill-labs/windmill/compare/v1.14.0...v1.14.1) (2022-06-27)
+
+
+### Bug Fixes
+
+* internal state for script triggers v1 ([6321311](https://github.com/windmill-labs/windmill/commit/6321311112dfa3ef09447f41847b248c0e0dcb46))
+
+## [1.14.0](https://github.com/windmill-labs/windmill/compare/v1.13.0...v1.14.0) (2022-06-27)
+
+
+### Features
+
+* add tesseract bin to worker image ([6de9697](https://github.com/windmill-labs/windmill/commit/6de9697d955a06cfb9c64fdb501b4dfa1bb597ad))
+* deno run with --unstable ([4947661](https://github.com/windmill-labs/windmill/commit/4947661b1d91867c022bb8a10a4be3e91f69352c))
+* internal state for script triggers mvp ([dcdb989](https://github.com/windmill-labs/windmill/commit/dcdb989adb8350974289a0c8d2239b245a6e0d41))
+
+
+### Bug Fixes
+
+* change default per page to 100 ([fdf95a0](https://github.com/windmill-labs/windmill/commit/fdf95a065e83d733ab6a0f02edb4af16c0a1dfb9))
+* deno exit after result logging ([6c622bc](https://github.com/windmill-labs/windmill/commit/6c622bcc32473361e1f7cb1ea7b0b508929bc1b8))
+* improve error handling ([f98f642](https://github.com/windmill-labs/windmill/commit/f98f6429c1e646c0a836f2f73a03a803aa655583))
+* improve error handling ([2efaf21](https://github.com/windmill-labs/windmill/commit/2efaf2191551c1406618c6d60bd37ca6eff84560))
+* schemaPicker does not display editor by default ([fc0c38f](https://github.com/windmill-labs/windmill/commit/fc0c38ffad18a9ceda44cb8406736c14ba4eb4c2))
+* smart assistant reload ([bb946ed](https://github.com/windmill-labs/windmill/commit/bb946ed5519f59adc559d6959c56e61403389c9d))
+
+## [1.13.0](https://github.com/windmill-labs/windmill/compare/v1.12.0...v1.13.0) (2022-06-22)
+
+
+### Features
+
+* better type narrowing for list and array types ([276319d](https://github.com/windmill-labs/windmill/commit/276319d99240dbca5bcc74a1142d99ca823c4da2))
+
+
+### Bug Fixes
+
+* fix webhook path for flows ([906f740](https://github.com/windmill-labs/windmill/commit/906f740a0ddce26743e4669af7a101613131a17c))
+* make email constraint case insensitive ([6dc90a3](https://github.com/windmill-labs/windmill/commit/6dc90a390643fcf6116289596ca1c3149d326797))
+
+## [1.12.0](https://github.com/windmill-labs/windmill/compare/v1.11.0...v1.12.0) (2022-06-14)
+
+
+### Bug Fixes
+
+* more flexible ResourceType MainArgSignature parser ([359ef15](https://github.com/windmill-labs/windmill/commit/359ef15fa2a9024507a71f2c656373925fba3ebe))
+* rename ResourceType -> Resource ([28b5671](https://github.com/windmill-labs/windmill/commit/28b56714023ea69a20f003e08f6c40de64202ac5))
+
+## [1.11.0](https://github.com/windmill-labs/windmill/compare/v1.10.1...v1.11.0) (2022-06-13)
+
+
+### Features
+
+* add DISABLE_NUSER for older kernels ([cce46f9](https://github.com/windmill-labs/windmill/commit/cce46f94404ac5c10407e430fff8cdec3bd7fb2d))
+* add ResourceType<'name'> as deno signature arg type ([f1ee5f3](https://github.com/windmill-labs/windmill/commit/f1ee5f3130cb7b753ccc3ee62169c5e4a8ef7b8b))
+
+
+### Bug Fixes
+
+* force c_ prefix for adding resource type ([9f235c4](https://github.com/windmill-labs/windmill/commit/9f235c404ed62b54a73451b9f9dbddd8f013120d))
+* **frontend:** loadItems not called in script picker ([a59b927](https://github.com/windmill-labs/windmill/commit/a59b92706b24a07cc14288620a9bcdb9402bd134))
+
+## [1.10.1](https://github.com/windmill-labs/windmill/compare/v1.10.0...v1.10.1) (2022-06-12)
+
+
+### Bug Fixes
+
+* python-client verify ssl ([295e28f](https://github.com/windmill-labs/windmill/commit/295e28fd43ef07b739d2c7c85b0ae6819f7d7434))
+
+## [1.10.0](https://github.com/windmill-labs/windmill/compare/v1.9.0...v1.10.0) (2022-06-11)
+
+
+### Features
+
+* alpha hub integration + frontend user store fixes + script client base_url fix ([1a61d50](https://github.com/windmill-labs/windmill/commit/1a61d50076b295fe97e48c2a621dff30802152b1))
+
+## [1.9.0](https://github.com/windmill-labs/windmill/compare/v1.8.6...v1.9.0) (2022-06-05)
+
+
+### Features
+
+* update postgres 13->14 in docker-compose ([479a12f](https://github.com/windmill-labs/windmill/commit/479a12f33ca26bfd1b67bcdd24a64ca26cc6bebe))
+
+
+### Bug Fixes
+
+* remove annoying transitions for scripts and flows ([f2348b5](https://github.com/windmill-labs/windmill/commit/f2348b5526bb8197519685cb57049f74c6f3a11d))
+
+### [1.8.6](https://github.com/windmill-labs/windmill/compare/v1.8.5...v1.8.6) (2022-05-18)
+
+
+### Bug Fixes
+
+* re-release ([d31cd3c](https://github.com/windmill-labs/windmill/commit/d31cd3c52c1b46e821da261f22d0aec872b61fb2))
+
+### [1.8.5](https://github.com/windmill-labs/windmill/compare/v1.8.4...v1.8.5) (2022-05-18)
+
+
+### Bug Fixes
+
+* language field broke flow too ([33fed8e](https://github.com/windmill-labs/windmill/commit/33fed8e04d3abbde371535ecb6e7ba15d103db92))
+
+### [1.8.4](https://github.com/windmill-labs/windmill/compare/v1.8.3...v1.8.4) (2022-05-18)
+
+
+### Bug Fixes
+
+* scripts run was broken due to 1.7 and 1.8 changes. This fix it ([7564d2c](https://github.com/windmill-labs/windmill/commit/7564d2cb1e7f600ede22f333a02a537df381d829))
+
+### [1.8.3](https://github.com/windmill-labs/windmill/compare/v1.8.2...v1.8.3) (2022-05-18)
+
+
+### Bug Fixes
+
+* clean exported deno-client api ([605c2b4](https://github.com/windmill-labs/windmill/commit/605c2b4d11bf072332a38f0c3e24cf6cc9ec7e65))
+
+### [1.8.2](https://github.com/windmill-labs/windmill/compare/v1.8.1...v1.8.2) (2022-05-18)
+
+
+### Bug Fixes
+
+* deno client ([563ba3e](https://github.com/windmill-labs/windmill/commit/563ba3e7f763279a93f619933ac35a1dec3f727a))
+* deno lsp client ([3eed59f](https://github.com/windmill-labs/windmill/commit/3eed59fcb1b172ab13f65c9a0caa0545f5ed91da))
+* deno lsp uses wss instead of ws ([865d728](https://github.com/windmill-labs/windmill/commit/865d728224bed55fe4a2c1905ff2b8c15f4bbe17))
+* starting deno script is now async ([7365a8e](https://github.com/windmill-labs/windmill/commit/7365a8e87bdb1f879eb92125a9e6378a1636637e))
+
+### [1.8.1](https://github.com/windmill-labs/windmill/compare/v1.8.0...v1.8.1) (2022-05-17)
+
+
+### Bug Fixes
+
+* frontend dependencies update ([f793bc4](https://github.com/windmill-labs/windmill/commit/f793bc46d98349a5fea56c7911b6e0720b2b117c))
+
+## [1.8.0](https://github.com/windmill-labs/windmill/compare/v1.7.0...v1.8.0) (2022-05-17)
+
+
+### Features
+
+* Typescript support for scripts (alpha) ([2e1d430](https://github.com/windmill-labs/windmill/commit/2e1d43033f3ad6dbe86338b7a41da7b1120a5ffc))
+
+## [1.7.0](https://github.com/windmill-labs/windmill/compare/v1.6.1...v1.7.0) (2022-05-14)
+
+
+### Features
+
+* self host github oauth ([#46](https://github.com/windmill-labs/windmill/issues/46)) ([5b413d7](https://github.com/windmill-labs/windmill/commit/5b413d7e045d09dc5c5916cb22d82438ec6c92ad))
+
+
+### Bug Fixes
+
+* better error message when saving script ([02c8bea](https://github.com/windmill-labs/windmill/commit/02c8bea0840e492c31ccb8ddd1e5ae9676a534b1))
+
+### [1.6.1](https://github.com/windmill-labs/windmill/compare/v1.6.0...v1.6.1) (2022-05-10)
+
+
+### Bug Fixes
+
+* also store and display "started at" for completed jobs ([#33](https://github.com/windmill-labs/windmill/issues/33)) ([2c28031](https://github.com/windmill-labs/windmill/commit/2c28031e44453740ad8c4b7e3c248173eab34b9c))
+
+## 1.6.0 (2022-05-10)
+
+### Features
+
+* superadmin settings ([7a51f84](https://www.github.com/windmill-labs/windmill/commit/7a51f842f01e17c4d230c060fa0de558553ad3ed))
+* user settings is now at workspace level ([a130806](https://www.github.com/windmill-labs/windmill/commit/a130806e1929267ee40ca443e3dac6e1a5d80da3))
+
+
+### Bug Fixes
+
+* display more than default 30 workspaces as superadmin ([55b5695](https://www.github.com/windmill-labs/windmill/commit/55b5695673912ffe040d3011c020b1002b4e3268))
+
 ## [1.5.0](https://www.github.com/windmill-labs/windmill/v1.5.0) (2022-05-02)

CLA.md

@@ -0,0 +1,145 @@
+## Contributor Agreement
+
+## Individual Contributor Non-Exclusive License Agreement
+
+Thank you for your interest in contributing to Windmill Labs, Inc's Windmill
+("We" or "Us").
+
+The purpose of this contributor agreement ("Agreement") is to clarify and
+document the rights granted by contributors to Us.
+
+### 1\. Definitions
+
+**"You"** means the individual Copyright owner who Submits a Contribution to Us.
+
+**"Legal Entity"** means an entity that is not a natural person.
+
+**"Affiliate"** means any other Legal Entity that controls, is controlled by, or
+under common control with that Legal Entity. For the purposes of this
+definition, "control" means (i) the power, direct or indirect, to cause the
+direction or management of such Legal Entity, whether by contract or otherwise,
+(ii) ownership of fifty percent (50%) or more of the outstanding shares or
+securities that vote to elect the management or other persons who direct such
+Legal Entity or (iii) beneficial ownership of such entity.
+
+**"Contribution"** means any original work of authorship, including any original
+modifications or additions to an existing work of authorship, Submitted by You
+to Us, in which You own the Copyright.
+
+**"Copyright"** means all rights protecting works of authorship, including
+copyright, moral and neighboring rights, as appropriate, for the full term of
+their existence.
+
+**"Material"** means the software or documentation made available by Us to third
+parties.
+
+**"Submit"** means any act by which a Contribution is transferred to Us by You
+by means of tangible or intangible media, including but not limited to
+electronic mailing lists, source code control systems, and issue tracking
+systems that are managed by, or on behalf of, Us, but excluding any transfer
+that is conspicuously marked or otherwise designated in writing by You as "Not a
+Contribution."
+
+**"Documentation"** means any non-software portion of a Contribution.
+
+### 2\. License grant
+
+#### 2.1 Copyright license to Us
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Us a
+worldwide, royalty-free, NON-exclusive, perpetual and irrevocable (except as
+stated in Section 8.2) license, with the right to transfer an unlimited number
+of non-exclusive licenses or to grant sublicenses to third parties, under the
+Copyright covering the Contribution to use the Contribution by all means,
+including, but not limited to:
+
+- publish the Contribution,
+- modify the Contribution,
+- prepare derivative works based upon or containing the Contribution and/or to
+  combine the Contribution with other Materials,
+- reproduce the Contribution in original or modified form,
+- distribute, to make the Contribution available to the public, display and
+  publicly perform the Contribution in original or modified form.
+
+#### 2.2 Moral rights
+
+Moral Rights remain unaffected to the extent they are recognized and not
+waivable by applicable law. Notwithstanding, You may add your name to the
+attribution mechanism customary used in the Materials you Contribute to, such as
+the header of the source code files of Your Contribution, and We will respect
+this attribution when using Your Contribution.
+
+### 3\. Patents
+
+#### 3.1 Patent license
+
+Subject to the terms and conditions of this Agreement You hereby grant to Us and
+to recipients of Materials distributed by Us a worldwide, royalty-free,
+non-exclusive, perpetual and irrevocable (except as stated in Section 3.2)
+patent license, with the right to transfer an unlimited number of non-exclusive
+licenses or to grant sublicenses to third parties, to make, have made, use,
+sell, offer for sale, import and otherwise transfer the Contribution and the
+Contribution in combination with any Material (and portions of such
+combination). This license applies to all patents owned or controlled by You,
+whether already acquired or hereafter acquired, that would be infringed by
+making, having made, using, selling, offering for sale, importing or otherwise
+transferring of Your Contribution(s) alone or by combination of Your
+Contribution(s) with any Material.
+
+### 4. Disclaimer
+
+THE CONTRIBUTION IS PROVIDED "AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED
+WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF SATISFACTORY
+QUALITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY
+DISCLAIMED BY YOU TO US AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES
+CANNOT BE DISCLAIMED, SUCH WARRANTY IS LIMITED IN DURATION AND EXTENT TO THE
+MINIMUM PERIOD AND EXTENT PERMITTED BY LAW.
+
+### 5. Consequential damage waiver
+
+TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR WE BE
+LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA,
+INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT
+OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR
+OTHERWISE) UPON WHICH THE CLAIM IS BASED.
+
+### 6. Approximation of disclaimer and damage waiver
+
+IF THE DISCLAIMER AND DAMAGE WAIVER MENTIONED IN SECTION 4. AND SECTION 5.
+CANNOT BE GIVEN LEGAL EFFECT UNDER APPLICABLE LOCAL LAW, REVIEWING COURTS SHALL
+APPLY LOCAL LAW THAT MOST CLOSELY APPROXIMATES AN ABSOLUTE WAIVER OF ALL CIVIL
+OR CONTRACTUAL LIABILITY IN CONNECTION WITH THE CONTRIBUTION.
+
+### 7. Term
+
+7.1 This Agreement shall come into effect upon Your acceptance of the terms and
+conditions.
+
+7.3 In the event of a termination of this Agreement Sections 4, 5, 6, 7 and 8
+shall survive such termination and shall remain in full force thereafter. For
+the avoidance of doubt, Free and Open Source Software (sub)licenses that have
+already been granted for Contributions at the date of the termination shall
+remain in full force after the termination of this Agreement.
+
+### 8 Miscellaneous
+
+8.1 This Agreement and all disputes, claims, actions, suits or other proceedings
+arising out of this agreement or relating in any way to it shall be governed by
+the laws of France excluding its private international law provisions.
+
+8.2 This Agreement sets out the entire agreement between You and Us for Your
+Contributions to Us and overrides all other agreements or understandings.
+
+8.3 In case of Your death, this agreement shall continue with Your heirs. In
+case of more than one heir, all heirs must exercise their rights through a
+commonly authorized person.
+
+8.4 If any provision of this Agreement is found void and unenforceable, such
+provision will be replaced to the extent possible with a provision that comes
+closest to the meaning of the original provision and that is enforceable. The
+terms and conditions set forth in this Agreement shall apply notwithstanding any
+failure of essential purpose of this Agreement or any limited remedy to the
+maximum extent possible under law.
+
+8.5 You agree to notify Us of any facts or circumstances of which you become
+aware that would make this Agreement inaccurate in any respect.

Caddyfile

@@ -1,4 +0,0 @@
-{$SITE_URL} {  
-  bind {$ADDRESS}
-  reverse_proxy /* server:8000
-}

Dockerfile

@@ -19,7 +19,7 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
 RUN make
 
-FROM mhart/alpine-node:14 as frontend
+FROM node:18-alpine as frontend
 
 # install dependencies
 WORKDIR /frontend
@@ -30,8 +30,11 @@ RUN npm ci
 COPY frontend .
 RUN mkdir /backend
 COPY /backend/openapi.yaml /backend/openapi.yaml
+COPY /openflow.openapi.yaml /openflow.openapi.yaml
 RUN npm run generate-backend-client
+ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
+RUN npm run check
 
 FROM rust:slim-buster as builder
 
@@ -56,11 +59,11 @@ RUN rm src/*.rs
 RUN rm ./target/release/deps/windmill*
 ENV SQLX_OFFLINE=true
 
-ADD ./backend ./
-ADD ./nsjail /nsjail
+COPY ./backend ./
+COPY ./nsjail /nsjail
 
-COPY --from=1 /frontend /frontend
-ADD .git/ .git/
+COPY --from=frontend /frontend /frontend
+COPY .git/ .git/
 
 RUN cargo build --release
 
@@ -69,11 +72,11 @@ FROM debian:buster-slim
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 python3 python3-pip \
+    && apt-get install -y ca-certificates tzdata libpq5 \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev \
+    libv8-dev tesseract-ocr \
     && rm -rf /var/lib/apt/lists/*
 
 ENV TZ=Etc/UTC
@@ -84,12 +87,14 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
     && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
     && make -j 4 && make install
 
-RUN python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install pip-tools
 
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -2,7 +2,7 @@
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
 
-Every file is under copyright (c) Ruben Fiszel 2021 unless otherwise specified.
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 

LICENSE-APACHE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2021 Ruben Fiszel
+   Copyright 2022 Windmill Labs, Inc
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -1,6 +1,4 @@
-Ruben Fiszel
-
-Copyright (c) 2021  Ruben Fiszel
+Copyright (c) 2022  Windmill Labs, Inc
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0 or the GNU Affero General Public License. Please see

README.md

@@ -1,10 +1,13 @@
 <p align="center">
-  <a href="https://alpha.windmill.dev"><img src="./windmill.svg" alt="windmill.dev"></a>
+  <a href="https://app.windmill.dev"><img src="./imgs/windmill.svg" alt="windmill.dev"></a>
 </p>
 <p align="center">
-    <em>Windmill.dev is an OSS developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
+    <em>Windmill is an open-source developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
 </p>
 <p align="center">
+<a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
+    <img src="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml/badge.svg" alt="Docker Image CI">
+</a>
 <a href="https://pypi.org/project/wmill" target="_blank">
     <img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
@@ -15,20 +18,25 @@
 
 ---
 
-**Join the alpha (personal workspaces are free forever)**:
-<https://alpha.windmill.dev>
+**Join the beta (personal workspaces are free forever)**:
+<https://app.windmill.dev>
 
 **Documentation**: <https://docs.windmill.dev>
 
 **Discord**: <https://discord.gg/V7PM2YHsPB>
 
-**We are hiring**: Software Engineers, DevOps, Solutions Engineers, Growth:
-<https://docs.windmill.dev/hiring>
+**Hub**: <https://hub.windmill.dev>
 
 You can show your support for the project by starring this repo.
 
 ---
 
+If you would like to run this in production self-hosted, or know someone that
+would, Windmill would gladly help you to achieve it and more. If interested,
+send an email to ruben@windmill.dev (founder and creator of Windmill).
+
+---
+
 # Windmill
 
 <p align="center">
@@ -36,17 +44,36 @@ You can show your support for the project by starring this repo.
 especially concerning flows.
 </p>
 
-![Windmill](./windmill.webp)
+![Windmill Screenshot](./imgs/windmill.webp)
 
 Windmill is <b>fully open-sourced</b>:
 
-- `community/` and `python-client/` are Apache 2.0
+- `community/`, `python-client/` and `deno-client/` are Apache 2.0
 - backend, frontend and everything else under AGPLv3.
 
+## What is the general idea behind Windmill
+
+1. Define a minimal and generic script in Python or Typescript that solve a
+   specific task. Here sending an email with SMTP. The code can be defined in
+   the provided Web IDE or synchronized with your own github repo:
+   ![Step 1](./imgs/step1.png)
+
+2. Your scripts parameters are automatically parsed and generate a frontend. You
+   can narrow down the types during task definition to specify regex for string,
+   an enum or a specific format for objects. Each script correspond to an app by
+   itself: ![Step 2](./imgs/step2.png)
+
+3. Make it flow! You can chain your scripts or scripts made by the community
+   inside flow by piping output to input using "Dynamic" fields that are just
+   plain Javascript. You can also refer to external variables, output from any
+   steps or inputs of the flow itself. The flow parameters then generate
+   automatically an intuitive forms that can be triggered by anyone, like for
+   scripts. ![Step 3](./imgs/step3.png)
+
 ## Layout
 
 - `backend/`: The whole Rust backend
-- `frontend`: The whole Svelte fronten
+- `frontend`: The whole Svelte frontend
 - `community/`: Scripts and resource types created and curated by the community,
   included in every workspace
 - `lsp/`: The lsp asssistant for the monaco editor
@@ -54,42 +81,74 @@ Windmill is <b>fully open-sourced</b>:
   execution
 - `python-client/`: The wmill python client used within scripts to interact with
   the windmill platform
+- `deno-client/`: The wmill deno client used within scripts to interact with the
+  windmill platform
 
 ## Stack
 
 - postgres as the database
-- backend in Rust with the follwing highly-available and horizontally scalable
+- backend in Rust with the following highly-available and horizontally scalable
   architecture:
   - stateless API backend
   - workers that pull jobs from a queue
-- frontend in svelte
+- frontend in Svelte
 - scripts executions are sandboxed using google's nsjail
 - javascript runtime is deno_core rust library (which itself uses the rusty_v8
   and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
 
-## Architecture
-
-A detailed section about Windmill architecture is coming soon
-
 ### Development stack
 
 - caddy is the reverse proxy used for local development, see frontend's
   Caddyfile and CaddyfileRemote
 
+## Architecture
+
+![Architecture](./imgs/architecture.svg)
+
 ## How to self-host
 
-Complete instructions coming soon
+`docker compose up` with the following docker-compose is sufficient:
+<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
+
+For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
+nsjail will not be able to launch the isolated scripts.
+
+To disable nsjail altogether, set `DISABLE_NSJAIL=true`.
+
+The default super-admin user is: admin@windmill.dev / changeme
+
+From there, you can create other users (do not forget to change the password!)
+
+Detailed instructions for more complex deployments will come soon. For simpler
+docker based ones, the docker-compose.yml file contains all the necessary
+informations.
+
+### OAuth for self-hosting
+
+To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
+the following format:
+
+```json
+{
+  "<client>":
+    "id": "<CLIENT_ID>",
+    "secret": "<CLIENT_SECRET>"
+}
+```
+
+and mount it at `/src/usr/app/oauth.json`.
+
+You will also want to import all the approved resource types from
+[WindmillHub](https://hub.windmill.dev).
+
+## Contributors
+
+<a href="https://github.com/windmill-labs/windmill/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=windmill-labs/windmill" />
+</a>
 
 ## Copyright
 
-2021 [Ruben Fiszel](https://github.com/rubenfiszel)
-
-### Acknowledgement
-
-This project is inspired from a previous project called
-[Delightool](https://github.com/windmill-labs/delightool-legacy) which was also
-led by [Ruben](https://github.com/rubenfiszel) and with large contribution on
-the frontend from [Malo Marrec](https://github.com/malomarrec) who gave his
-blessing to Windmill.
+Windmill Labs, Inc 2022

backend/.gitattributes

@@ -0,0 +1 @@
+sqlx-data.json -diff

backend/.gitignore

@@ -1,2 +1,3 @@
 target/
 .env
+oauth.json

backend/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "windmill"
-version = "1.5.0"
-authors = ["Ruben Fiszel <ruben@rubenfiszel.com>"]
+version = "1.27.2"
+authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
 [build-dependencies]
@@ -17,7 +17,7 @@ tower-http = { version = "^0", features = ["trace"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
-uuid = { version = "^0", features = ["serde", "v4"] }
+uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"]}
@@ -31,30 +31,32 @@ hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
 retainer = "^0"
-rand = "^0.8.4"
-rand_core = { version = "^0.6.3", features = ["std"] }
+rand = "^0"
+rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
 rustpython-parser = "^0"
 cron = "^0"
 external-ip = "^4"
-lettre = { version = "^0.10.0-rc.4", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
+lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
 urlencoding = "^2"
-oauth2 = "^4"
 url = "^2"
+async-oauth2 = "^0"
 reqwest = { version = "^0", features = ["json"] }
-time = "0.3.7"
+time = "^0"
 slack-http-verifier = "^0"
 serde_urlencoded = "^0"
 tokio-tar = "^0"
 tempfile = "^3"
-tokio-util = { version = "0.7.0", features = ["io"] }
+tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 deno_core = "^0"
-indexmap = "~1.6.2"
 async-recursion = "^1"
+swc_common = "^0"
+swc_ecma_parser = "^0"
+swc_ecma_ast = "^0"
 
 sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
 dotenv = "^0"

backend/build.rs

@@ -1,17 +0,0 @@
-use std::fs::File;
-use std::io::Write;
-
-use deno_core::{JsRuntime, RuntimeOptions};
-
-fn main() {
-    println!("cargo:rerun-if-changed=build.rs");
-    let options = RuntimeOptions {
-        will_snapshot: true,
-        ..Default::default()
-    };
-    let mut runtime = JsRuntime::new(options);
-
-    let mut snap = File::create("v8.snap").expect("can create snap file");
-    snap.write_all(&runtime.snapshot())
-        .expect("can write content to snap");
-}

backend/migrations/20220321004844_flow_preview.down.sql

@@ -1 +1,2 @@
 -- Add down migration script here
+

backend/migrations/20220504193929_typescript_support.down.sql

@@ -0,0 +1,12 @@
+-- Add down migration script here
+DROP TYPE SCRIPT_LANG;
+
+ALTER TABLE script
+DROP COLUMN language SCRIPT_LANG;
+
+ALTER TABLE queue
+DROP COLUMN language SCRIPT_LANG;
+
+ALTER TABLE completed_job
+DROP COLUMN language SCRIPT_LANG;
+

backend/migrations/20220504193929_typescript_support.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+CREATE TYPE SCRIPT_LANG AS ENUM ('python3', 'deno');
+
+ALTER TABLE script
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
+
+ALTER TABLE queue
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
+
+ALTER TABLE completed_job
+ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';

backend/migrations/20220508150023_changeme_passwords.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508150023_changeme_passwords.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+UPDATE password 
+SET password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'
+WHERE password_hash = '$argon2id$v=19$m=4096,t=3,p=1$z0Kg3qyaS14e+YHeihkJLQ$N69flI6yQ/U98pjAHtbNxbdz2f4PrJEi9Tx1VoYk1as';

backend/migrations/20220508152326_create_role_if_not_exists.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508152326_create_role_if_not_exists.up.sql

@@ -0,0 +1,24 @@
+-- Add up migration script here
+DO
+$do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'app') THEN
+        CREATE ROLE app LOGIN PASSWORD 'changeme';
+    END IF;
+END
+$do$;
+
+DO
+$do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = 'admin') THEN
+        CREATE ROLE admin LOGIN PASSWORD 'changeme';
+    END IF;
+END
+$do$;

backend/migrations/20220508155403_delete_starter_scripts.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220508155403_delete_starter_scripts.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+DELETE FROM script WHERE lock IS NULL;
+

backend/migrations/20220509191845_started_at_completed_job.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE completed_job
+DROP COLUMN started_at;

backend/migrations/20220509191845_started_at_completed_job.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE completed_job
+ADD COLUMN started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW();
+

backend/migrations/20220518211432_make_lang_nullable.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220518211432_make_lang_nullable.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+ALTER TABLE queue
+ALTER COLUMN language DROP NOT NULL;
+
+ALTER TABLE completed_job
+ALTER COLUMN language DROP NOT NULL;

backend/migrations/20220610181005_add_script_hub.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220610181005_add_script_hub.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE JOB_KIND ADD VALUE 'script_hub';

backend/migrations/20220615151034_accounts.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220615151034_accounts.up.sql

@@ -0,0 +1,13 @@
+-- Add up migration script here
+
+CREATE TABLE account (
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    id SERIAL NOT NULL,
+    expires_at TIMESTAMP,
+    refresh_token VARCHAR(255),
+    PRIMARY KEY (workspace_id, id)
+);
+
+ALTER TABLE resource ADD COLUMN account INTEGER;
+ALTER TABLE variable ADD COLUMN account INTEGER;
+ALTER TABLE password ALTER COLUMN login_type TYPE VARCHAR(50);

backend/migrations/20220620210708_regex_fix.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220620210708_regex_fix.up.sql

@@ -0,0 +1,9 @@
+-- Add up migration script here
+
+ALTER TABLE usr DROP CONSTRAINT proper_email;
+ALTER TABLE usr ADD CONSTRAINT proper_email
+CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');
+
+ALTER TABLE workspace_invite DROP CONSTRAINT proper_email;
+ALTER TABLE workspace_invite ADD CONSTRAINT proper_email
+CHECK (email ~* '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$');

backend/migrations/20220624170622_workspace_premium.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220624170622_workspace_premium.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace ADD COLUMN premium BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20220624180013_workspace_premium_grant.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220624180013_workspace_premium_grant.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT SELECT ON workspace TO app;
+

backend/migrations/20220707094935_add_client_to_account.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220707094935_add_client_to_account.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+ALTER TABLE account ADD COLUMN owner VARCHAR(50) NOT NULL;
+ALTER TABLE account ADD COLUMN client VARCHAR(50) NOT NULL;
+ALTER TABLE resource ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE variable ADD COLUMN is_oauth BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE resource DROP COLUMN account;
+
+ALTER TABLE account ALTER COLUMN expires_at TYPE TIMESTAMP WITH TIME ZONE;
+
+ALTER TABLE account ALTER COLUMN expires_at SET NOT NULL;
+ALTER TABLE account ALTER COLUMN refresh_token SET NOT NULL;
+
+GRANT ALL ON account TO app;
+GRANT ALL ON account TO admin;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO admin;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO app;
+
+ALTER TABLE account ENABLE ROW LEVEL SECURITY;
+
+
+CREATE POLICY see_own ON account FOR ALL
+USING (SPLIT_PART(account.owner, '/', 1) = 'u' AND SPLIT_PART(account.owner, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON account FOR ALL
+USING (SPLIT_PART(account.owner, '/', 1) = 'g' AND SPLIT_PART(account.owner, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20220713142758_script_trigger.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220713142758_script_trigger.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN trigger_reco_interval INTEGER;
+ALTER TABLE completed_job ADD COLUMN is_skipped BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20220719101639_script_trigger_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220719101639_script_trigger_2.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE script DROP COLUMN trigger_reco_interval;
+ALTER TABLE script ADD COLUMN is_trigger BOOLEAN NOT NULL DEFAULT false;
+

backend/migrations/20220728222351_job_duration_ms.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220728222351_job_duration_ms.up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE completed_job
+     RENAME duration to duration_ms;
+UPDATE completed_job
+   SET duration_ms = duration_ms * 1000;

backend/oauth_connect.json

@@ -0,0 +1,73 @@
+{
+    "github": {
+        "auth_url": "https://github.com/login/oauth/authorize",
+        "token_url": "https://github.com/login/oauth/access_token",
+        "scopes": [
+            "workflow",
+            "repo"
+        ]
+    },
+    "gitlab": {
+        "auth_url": "https://gitlab.com/oauth/authorize",
+        "token_url": "https://gitlab.com/oauth/token",
+        "scopes": [
+            "api"
+        ]
+    },
+    "bitbucket": {
+        "auth_url": "https://bitbucket.org/site/oauth2/authorize",
+        "token_url": "https://bitbucket.org/site/oauth2/access_token",
+        "scopes": [
+            "repository"
+        ]
+    },
+    "slack": {
+        "auth_url": "https://slack.com/oauth/authorize",
+        "token_url": "https://slack.com/api/oauth.access",
+        "scopes": [
+            "chat:write:user"
+        ]
+    },
+    "gsheets": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/spreadsheets"
+        ]
+    },
+    "gdrive": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/drive"
+        ]
+    },
+    "gmail": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://mail.google.com/"
+        ]
+    },
+    "gcal": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/calendar.events"
+        ]
+    },
+    "gcloud": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform"
+        ]
+    },
+    "discord": {
+        "auth_url": "https://discordapp.com/api/oauth2/authorize",
+        "token_url": "https://discordapp.com/api/oauth2/token",
+        "scopes": [
+            "bot"
+        ]
+    }
+}

backend/oauth_login.json

@@ -0,0 +1,23 @@
+{
+    "github": {
+        "auth_url": "https://github.com/login/oauth/authorize",
+        "token_url": "https://github.com/login/oauth/access_token",
+        "scopes": [
+            "user:email"
+        ]
+    },
+    "gitlab": {
+        "auth_url": "https://gitlab.com/oauth/authorize",
+        "token_url": "https://gitlab.com/oauth/token",
+        "scopes": [
+            "read_user"
+        ]
+    },
+    "google": {
+        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+        "token_url": "https://oauth2.googleapis.com/token",
+        "scopes": [
+            "https://www.googleapis.com/auth/userinfo.email"
+        ]
+    }
+}

backend/rustfmt.toml

@@ -1,7 +1,5 @@
-imports_granularity = "Crate"
 max_width = 100
 use_small_heuristics = "Default"
-indent_style = "Block"
-fn_single_line = false
-force_multiline_blocks = true
-format_strings = true
+match_arm_leading_pipes="Preserve"
+struct_lit_width=100
+struct_variant_width=100

backend/src/audit.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -64,13 +65,12 @@ pub async fn audit_log<'c>(
     let p_json: serde_json::Value = serde_json::to_value(&parameters).unwrap();
 
     tracing::info!(
-        username = username,
-        kind = "audit",
         operation = operation,
-        workspace = w_id,
         action_kind = ?action_kind,
         resource = resource,
-        parameters = %p_json
+        parameters = %p_json,
+        workspace_id = w_id,
+        username = username,
     );
     sqlx::query(
         "INSERT INTO audit

backend/src/client.rs

@@ -31,14 +31,17 @@ pub async fn get_resource(
     base_url: &str,
 ) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let client = reqwest::Client::new();
-    let result = client
+    let res = client
         .get(format!(
             "{base_url}/api/w/{workspace}/resources/get_value/{path}"
         ))
         .bearer_auth(token)
         .send()
-        .await?
-        .json::<Option<serde_json::Value>>()
         .await?;
-    Ok(result)
+    if res.status().is_success() {
+        let value = res.json::<Option<serde_json::Value>>().await?;
+        Ok(value)
+    } else {
+        Err(Error::NotFound(format!("Variable not found at {path}")))?
+    }
 }

backend/src/db.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.

backend/src/error.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -44,7 +45,7 @@ pub enum Error {
     HexErr(#[from] hex::FromHexError),
     #[error("Migrating database: {0}")]
     DatabaseMigration(#[from] MigrateError),
-    #[error("{0}")]
+    #[error(transparent)]
     Anyhow(#[from] anyhow::Error),
 }
 
@@ -62,6 +63,11 @@ impl IntoResponse for Error {
             Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST,
             _ => StatusCode::INTERNAL_SERVER_ERROR,
         };
-        Response::builder().status(status).body(body).unwrap()
+        tracing::error!(error = e.to_string());
+        Response::builder()
+            .header("Content-Type", "text/plain")
+            .status(status)
+            .body(body)
+            .unwrap()
     }
 }

backend/src/flows.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -7,24 +8,26 @@
 
 use std::collections::HashMap;
 
+use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
-    extract::{Extension, Path, Query},
+    extract::{Extension, Host, Path, Query},
     routing::{get, post},
     Json, Router,
 };
 use serde::{Deserialize, Serialize};
 use sql_builder::SqlBuilder;
-use sqlx::FromRow;
+use sqlx::{FromRow, Postgres, Transaction};
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::UserDB,
-    error::{Error, JsonResult, Result},
+    db::{UserDB, DB},
+    error::{self, to_anyhow, Error, JsonResult, Result},
+    jobs::RawCode,
     scripts::Schema,
     users::Authed,
-    utils::{Pagination, StripPath},
+    utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath},
 };
 
 pub fn workspaced_service() -> Router {
@@ -34,6 +37,13 @@ pub fn workspaced_service() -> Router {
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
+        .route("/exists/*path", get(exists_flow_by_path))
+}
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/hub/list", get(list_hub_flows))
+        .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
 #[derive(FromRow, Serialize)]
@@ -59,19 +69,21 @@ pub struct NewFlow {
     pub schema: Option<Schema>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct FlowValue {
     pub modules: Vec<FlowModule>,
     pub failure_module: Option<FlowModule>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct FlowModule {
     pub input_transform: HashMap<String, InputTransform>,
     pub value: FlowModuleValue,
+    pub stop_after_if_expr: Option<String>,
+    pub skip_if_stopped: Option<bool>,
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Debug, Clone)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
@@ -79,17 +91,31 @@ pub struct FlowModule {
 pub enum InputTransform {
     Static { value: serde_json::Value },
     Javascript { expr: String },
-    Resource { path: String },
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(
     tag = "type",
     rename_all(serialize = "lowercase", deserialize = "lowercase")
 )]
 pub enum FlowModuleValue {
-    Script { path: String },
-    Flow { path: String },
+    Script {
+        path: String,
+    },
+    ForloopFlow {
+        iterator: InputTransform,
+        value: Box<FlowValue>,
+        #[serde(default = "default_true")]
+        skip_failures: bool,
+    },
+    Flow {
+        path: String,
+    },
+    RawScript(RawCode),
+}
+
+fn default_true() -> bool {
+    true
 }
 
 #[derive(Deserialize)]
@@ -121,7 +147,7 @@ async fn list_flows(
             "edited_by",
             "edited_at",
             "archived",
-            "schema",
+            "null schema",
             "extra_perms",
         ])
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
@@ -150,6 +176,43 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
+async fn list_hub_flows(
+    Authed { email, username, .. }: Authed,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let flows = list_elems_from_hub(
+        http_client,
+        "https://hub.windmill.dev/searchFlowData?approved=true",
+        email,
+        username,
+        host,
+    )
+    .await?;
+    Ok(Json(flows))
+}
+
+pub async fn get_hub_flow_by_id(
+    Authed { email, username, .. }: Authed,
+    Path(id): Path<i32>,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let value = http_get_from_hub(
+        http_client,
+        &format!("https://hub.windmill.dev/flows/{id}/json"),
+        email,
+        username,
+        host,
+        false,
+    )
+    .await?
+    .json()
+    .await
+    .map_err(to_anyhow)?;
+    Ok(Json(value))
+}
+
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -159,8 +222,11 @@ async fn create_flow(
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
+    check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+
     sqlx::query!(
-        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::json)",
+        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
+         schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::json)",
         w_id,
         nf.path,
         nf.summary,
@@ -193,6 +259,29 @@ async fn create_flow(
     Ok(nf.path.to_string())
 }
 
+async fn check_schedule_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> error::Result<()> {
+    let exists_flow = sqlx::query_scalar!(
+        "SELECT EXISTS (SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2 AND path != \
+         script_path)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists_flow {
+        return Err(error::Error::BadConfig(format!(
+            "A flow cannot have the same path as a schedule if the schedule does not trigger that \
+             same flow: {path}",
+        )));
+    };
+    Ok(())
+}
+
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -202,9 +291,12 @@ async fn update_flow(
     let mut tx = user_db.begin(&authed).await?;
 
     let flow_path = flow_path.to_path();
+    check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
+
     let schema = nf.schema.map(|x| x.0);
     let flow = sqlx::query_scalar!(
-        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = $6, schema = $7 WHERE path = $8 AND workspace_id = $9 RETURNING path",
+        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
+         edited_at = $6, schema = $7 WHERE path = $8 AND workspace_id = $9 RETURNING path",
         nf.path,
         nf.summary,
         nf.description,
@@ -260,6 +352,25 @@ async fn get_flow_by_path(
     Ok(Json(flow))
 }
 
+async fn exists_flow_by_path(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id \
+         = 'starter'))",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -302,22 +413,46 @@ mod tests {
         let mut hm = HashMap::new();
         hm.insert(
             "test".to_owned(),
-            InputTransform::Static {
-                value: serde_json::json!("test2"),
-            },
+            InputTransform::Static { value: serde_json::json!("test2") },
         );
         let fv = FlowValue {
-            modules: vec![FlowModule {
-                input_transform: hm,
-                value: FlowModuleValue::Script {
-                    path: "test".to_string(),
+            modules: vec![
+                FlowModule {
+                    input_transform: hm,
+                    value: FlowModuleValue::Script { path: "test".to_string() },
+                    stop_after_if_expr: None,
+                    skip_if_stopped: Some(false),
                 },
-            }],
+                FlowModule {
+                    input_transform: HashMap::new(),
+                    value: FlowModuleValue::RawScript(RawCode {
+                        content: "test".to_string(),
+                        language: crate::scripts::ScriptLang::Deno,
+                        path: None,
+                    }),
+                    stop_after_if_expr: Some("foo = 'bar'".to_string()),
+                    skip_if_stopped: None,
+                },
+                FlowModule {
+                    input_transform: [(
+                        "iterand".to_string(),
+                        InputTransform::Static { value: serde_json::json!(vec![1, 2, 3]) },
+                    )]
+                    .into(),
+                    value: FlowModuleValue::ForloopFlow {
+                        iterator: InputTransform::Static { value: serde_json::json!([1, 2, 3]) },
+                        value: Box::new(FlowValue { modules: vec![], failure_module: None }),
+                        skip_failures: true,
+                    },
+                    stop_after_if_expr: Some("previous.res1.isEmpty()".to_string()),
+                    skip_if_stopped: None,
+                },
+            ],
             failure_module: Some(FlowModule {
                 input_transform: HashMap::new(),
-                value: FlowModuleValue::Flow {
-                    path: "test".to_string(),
-                },
+                value: FlowModuleValue::Flow { path: "test".to_string() },
+                stop_after_if_expr: Some("previous.res1.isEmpty()".to_string()),
+                skip_if_stopped: None,
             }),
         };
         println!("{}", serde_json::json!(fv).to_string());

backend/src/granular_acls.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -46,7 +47,8 @@ async fn add_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
+         true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(write.unwrap_or(false))
     .bind(path)
@@ -74,7 +76,8 @@ async fn remove_granular_acl(
 
     let identifier = if kind == "group_" { "name" } else { "path" };
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
-        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
+        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
+         workspace_id = $3 RETURNING extra_perms"
     ))
     .bind(owner)
     .bind(path)

backend/src/groups.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -258,9 +259,7 @@ async fn add_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username {
-        username: user_username,
-    }): Json<Username>,
+    Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
@@ -294,9 +293,7 @@ async fn remove_user(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
-    Json(Username {
-        username: user_username,
-    }): Json<Username>,
+    Json(Username { username: user_username }): Json<Username>,
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 

backend/src/js_eval.rs

@@ -1,57 +1,60 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::cell::RefCell;
-use std::rc::Rc;
-
-use deno_core::serde_v8;
-use deno_core::v8;
-use deno_core::v8::IsolateHandle;
-use deno_core::JsRuntime;
-use deno_core::OpState;
-use deno_core::RuntimeOptions;
-use deno_core::Snapshot;
-use deno_core::ZeroCopyBuf;
+use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
 use itertools::Itertools;
 use regex::Regex;
 use serde_json::Value;
-use tokio::sync::oneshot;
-use tokio::time::timeout;
+use tokio::{sync::oneshot, time::timeout};
 
-use crate::client;
-use crate::error::Error;
+use crate::{client, error::Error};
+
+pub struct EvalCreds {
+    pub workspace: String,
+    pub token: String,
+}
 
 pub async fn eval_timeout(
     expr: String,
     env: Vec<(String, serde_json::Value)>,
-    workspace: &str,
-    token: &str,
+    creds: Option<EvalCreds>,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr2 = expr.clone();
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
-    let (workspace, token) = (workspace.to_string().clone(), token.to_string().clone());
     timeout(
         std::time::Duration::from_millis(2000),
         tokio::task::spawn_blocking(move || {
-            let buffer = include_bytes!("../v8.snap");
+            let mut ops = vec![];
 
+            if creds.is_some() {
+                ops.extend([
+                    // An op for summing an array of numbers
+                    // The op-layer automatically deserializes inputs
+                    // and serializes the returned Result & value
+                    op_variable::decl(),
+                    op_resource::decl(),
+                ])
+            }
+
+            if !steps.is_empty() {
+                ops.push(op_get_result::decl())
+            }
+
+            let ext = Extension::builder().ops(ops).build();
             // Use our snapshot to provision our new runtime
             let options = RuntimeOptions {
-                startup_snapshot: Some(Snapshot::Static(buffer)),
+                extensions: vec![ext],
+                //                startup_snapshot: Some(Snapshot::Static(buffer)),
                 ..Default::default()
             };
+
             let mut js_runtime = JsRuntime::new(options);
-            js_runtime.register_op("variable", deno_core::op_async(op_variable));
-            js_runtime.register_op("resource", deno_core::op_async(op_resource));
-            if !steps.is_empty() {
-                js_runtime.register_op("result", deno_core::op_async(op_get_result));
-            }
-            js_runtime.sync_ops_cache();
 
             sender
                 .send(js_runtime.v8_isolate().thread_safe_handle())
@@ -68,8 +71,7 @@ pub async fn eval_timeout(
                 .into_iter()
                 .fold(expr, replace_with_await);
 
-            let r =
-                runtime.block_on(eval(&mut js_runtime, &expr, env, &workspace, &token, steps))?;
+            let r = runtime.block_on(eval(&mut js_runtime, &expr, env, creds, steps))?;
 
             Ok(r) as anyhow::Result<Value>
         }),
@@ -119,8 +121,7 @@ async fn eval(
     context: &mut JsRuntime,
     expr: &str,
     env: Vec<(String, serde_json::Value)>,
-    workspace: &str,
-    token: &str,
+    creds: Option<EvalCreds>,
     steps: Vec<String>,
 ) -> anyhow::Result<serde_json::Value> {
     let expr = expr.trim();
@@ -131,14 +132,12 @@ async fn eval(
             .join("\n"),
         expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
     );
-    let steps_code = if !steps.is_empty() {
-        format!(
-            r#"
+    let (steps_code, api_code) = if let Some(EvalCreds { workspace, token }) = creds {
+        let steps_code = if !steps.is_empty() {
+            format!(
+                r#"
 let steps = [{}];
 async function step(n) {{
-    if (n == 0) {{
-        return flow_input;
-    }}
     if (n == -1) {{
         return previous_result;
     }}
@@ -148,42 +147,55 @@ async function step(n) {{
         n = n % steps.length + steps.length;
     }}
     let id = steps[n];
-    return await Deno.core.opAsync("result", [workspace, id, token, base_url]);
+    return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
 }}"#,
-            steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
-        )
-    } else {
-        "".to_string()
-    };
+                steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
+            )
+        } else {
+            String::new()
+        };
 
-    let code = format!(
-        r#"
+        let api_code = format!(
+            r#"
 let workspace = "{workspace}";
 let base_url = "{}";
 async function variable(path) {{
     let token = "{token}";
-    return await Deno.core.opAsync("variable", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_variable", [workspace, path, token, base_url]);
 }}
 async function resource(path) {{
     let token = "{token}";
-    return await Deno.core.opAsync("resource", [workspace, path, token, base_url]);
+    return await Deno.core.opAsync("op_resource", [workspace, path, token, base_url]);
 }}
+        "#,
+            std::env::var("BASE_INTERNAL_URL")
+                .unwrap_or_else(|_| "http://missing-base-url".to_string()),
+        );
+        (steps_code, api_code)
+    } else {
+        (String::new(), String::new())
+    };
+
+    let code = format!(
+        r#"
+{api_code}
 {}
 {steps_code}
 (async () => {{ 
     {expr} 
 }})()
         "#,
-        std::env::var("BASE_INTERNAL_URL")
-            .unwrap_or_else(|_| "http://missing-base-url".to_string()),
         env.into_iter()
-            .map(|(a, b)| format!(
-                "let {a} = {};\n",
-                serde_json::to_string(&b)
-                    .unwrap_or_else(|_| "\"error serializing value\"".to_string())
-            ))
+            .map(|(a, b)| {
+                format!(
+                    "let {a} = {};\n",
+                    serde_json::to_string(&b)
+                        .unwrap_or_else(|_| "\"error serializing value\"".to_string())
+                )
+            })
             .join(""),
     );
+    tracing::debug!("{}", code);
     let global = context.execute_script("<anon>", &code)?;
     let global = context.resolve_value(global).await?;
 
@@ -204,11 +216,8 @@ async function resource(path) {{
 //     Ok(path)
 // }
 
-async fn op_variable(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<String, anyhow::Error> {
+#[op]
+async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -216,11 +225,8 @@ async fn op_variable(
     client::get_variable(workspace, path, token, &base_url).await
 }
 
-async fn op_get_result(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<Option<serde_json::Value>, anyhow::Error> {
+#[op]
+async fn op_get_result(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let id = &args[1];
     let token = &args[2];
@@ -238,11 +244,8 @@ async fn op_get_result(
     Ok(result)
 }
 
-async fn op_resource(
-    _state: Rc<RefCell<OpState>>,
-    args: Vec<String>,
-    _buf: Option<ZeroCopyBuf>,
-) -> Result<Option<serde_json::Value>, anyhow::Error> {
+#[op]
+async fn op_resource(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
     let workspace = &args[0];
     let path = &args[1];
     let token = &args[2];
@@ -267,7 +270,7 @@ mod tests {
         let code = "value.test + params.test";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
+        let res = eval(&mut runtime, code, env, None, vec![]).await?;
         assert_eq!(res, json!(4));
         Ok(())
     }
@@ -280,7 +283,7 @@ mod tests {
 multiline template`";
 
         let mut runtime = JsRuntime::new(RuntimeOptions::default());
-        let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
+        let res = eval(&mut runtime, code, env, None, vec![]).await?;
         assert_eq!(res, json!("my 5\nmultiline template"));
         Ok(())
     }
@@ -293,7 +296,7 @@ multiline template`";
         ];
         let code = r#"variable("test")"#;
 
-        let res = eval_timeout(code.to_string(), env, "workspace", "token", vec![]).await?;
+        let res = eval_timeout(code.to_string(), env, None, vec![]).await?;
         assert_eq!(res, json!("test"));
         Ok(())
     }

backend/src/lib.rs

@@ -1,24 +1,22 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
-use ::oauth2::basic::BasicClient;
 use argon2::Argon2;
-use axum::{extract::extractor_middleware, handler::Handler, routing::get, Extension, Router};
+use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
-use hyper::Response;
 use slack_http_verifier::SlackVerifier;
-use std::{collections::HashMap, net::SocketAddr, sync::Arc};
+use std::{net::SocketAddr, sync::Arc};
 use tokio::sync::Mutex;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
-use tower_http::trace::{MakeSpan, OnResponse, TraceLayer};
-use tracing::{field, Span};
-use tracing_subscriber::{filter::filter_fn, prelude::*, EnvFilter};
+use tower_http::trace::TraceLayer;
+
 extern crate magic_crypt;
 
 extern crate dotenv;
@@ -28,7 +26,7 @@ mod client;
 mod db;
 mod email;
 mod error;
-mod flow;
+mod flows;
 mod granular_acls;
 mod groups;
 mod jobs;
@@ -39,85 +37,31 @@ mod resources;
 mod schedule;
 mod scripts;
 mod static_assets;
+mod tracing_init;
 mod users;
 mod utils;
 mod variables;
 mod worker;
+mod worker_flow;
 mod worker_ping;
 mod workspaces;
 
 use error::Error;
 
 pub use crate::email::EmailSender;
-use crate::{db::UserDB, utils::rd_string};
+use crate::{
+    db::UserDB,
+    error::to_anyhow,
+    oauth2::build_oauth_clients,
+    tracing_init::{MyMakeSpan, MyOnResponse},
+    utils::rd_string,
+};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 pub const DEFAULT_NUM_WORKERS: usize = 3;
 pub const DEFAULT_TIMEOUT: i32 = 300;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
 
-#[derive(Clone)]
-struct MyOnResponse {}
-
-impl<B> OnResponse<B> for MyOnResponse {
-    fn on_response(
-        self,
-        response: &Response<B>,
-        latency: std::time::Duration,
-        _span: &tracing::Span,
-    ) {
-        tracing::info!(
-            latency = %latency.as_millis(),
-            status = ?response.status(),
-            "finished processed request")
-    }
-}
-
-#[derive(Clone)]
-struct MyMakeSpan {}
-
-impl<B> MakeSpan<B> for MyMakeSpan {
-    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
-        tracing::info_span!(
-            "request",
-            method = %request.method(),
-            uri = %request.uri(),
-            version = ?request.version(),
-            username = field::Empty,
-        )
-    }
-}
-
-pub async fn initialize_tracing() -> anyhow::Result<()> {
-    //let log_level = if std::env::var("RUST_LOG").map(|x| &x == "debug")
-    let ts_base = tracing_subscriber::registry()
-        .with(
-            EnvFilter::from_default_env()
-                //.add_directive("windmill".parse()?)
-                .add_directive("runtime=trace".parse()?)
-                .add_directive("tokio=trace".parse()?),
-        )
-        .with(
-            tracing_subscriber::fmt::layer()
-                .json()
-                .flatten_event(true)
-                .with_span_list(false)
-                .with_current_span(true)
-                .with_filter(filter_fn(|meta| meta.target().starts_with("windmill"))),
-        );
-
-    if std::env::var("TOKIO_CONSOLE")
-        .map(|x| x == "true")
-        .unwrap_or(false)
-    {
-        let console_layer = console_subscriber::spawn();
-        ts_base.with(console_layer).init();
-    } else {
-        ts_base.init();
-    }
-    Ok(())
-}
-
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
     let app_password = std::env::var("APP_USER_PASSWORD").unwrap_or_else(|_| "changeme".to_owned());
 
@@ -132,23 +76,14 @@ pub async fn connect_db() -> anyhow::Result<DB> {
     Ok(db::connect(&database_url).await?)
 }
 
-type BasicClientsMap = HashMap<String, BasicClient>;
-
-pub fn build_oauth_clients(base_url: &str) -> BasicClientsMap {
-    [(
-        "github".to_string(),
-        oauth2::build_gh_client(
-            &std::env::var("GITHUB_OAUTH_CLIENT_ID").unwrap_or_else(|_| "".to_string()),
-            &std::env::var("GITHUB_OAUTH_CLIENT_SECRET").unwrap_or_else(|_| "".to_string()),
-            base_url,
-        ),
-    )]
-    .into()
+pub async fn initialize_tracing() -> anyhow::Result<()> {
+    tracing_init::initialize_tracing().await
 }
 
-#[derive(Clone)]
 struct BaseUrl(String);
 
+struct CloudHosted(bool);
+
 pub async fn run_server(
     db: DB,
     addr: SocketAddr,
@@ -161,13 +96,16 @@ pub async fn run_server(
     let auth_cache = Arc::new(users::AuthCache::new(db.clone()));
     let argon2 = Arc::new(Argon2::default());
     let email_sender = Arc::new(es);
-    let basic_clients = Arc::new(build_oauth_clients(base_url));
+    let basic_clients = Arc::new(build_oauth_clients(base_url).await?);
     let slack_verifier = Arc::new(
         std::env::var("SLACK_SIGNING_SECRET")
             .ok()
             .map(|x| SlackVerifier::new(x).unwrap()),
     );
-
+    let http_client = reqwest::ClientBuilder::new()
+        .user_agent("windmill/beta")
+        .build()
+        .map_err(to_anyhow)?;
     let middleware_stack = ServiceBuilder::new()
         .layer(
             TraceLayer::new_for_http()
@@ -179,7 +117,11 @@ pub async fn run_server(
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(Extension(basic_clients))
-        .layer(Extension(BaseUrl(base_url.to_string())))
+        .layer(Extension(Arc::new(BaseUrl(base_url.to_string()))))
+        .layer(Extension(Arc::new(CloudHosted(
+            std::env::var("CLOUD_HOSTED").is_ok(),
+        ))))
+        .layer(Extension(http_client))
         .layer(CookieManagerLayer::new());
     // build our application with a route
     let app = Router::new()
@@ -205,7 +147,7 @@ pub async fn run_server(
                         .nest("/audit", audit::workspaced_service())
                         .nest("/acls", granular_acls::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/flows", flow::workspaced_service()),
+                        .nest("/flows", flows::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
@@ -214,9 +156,10 @@ pub async fn run_server(
                 )
                 .nest("/workers", worker_ping::global_service())
                 .nest("/scripts", scripts::global_service())
+                .nest("/flows", flows::global_service())
                 .nest("/schedules", schedule::global_service())
-                .route_layer(extractor_middleware::<users::Authed>())
-                .route_layer(extractor_middleware::<users::Tokened>())
+                .route_layer(from_extractor::<users::Authed>())
+                .route_layer(from_extractor::<users::Tokened>())
                 .nest(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),
@@ -265,6 +208,8 @@ pub async fn run_workers(
     num_workers: i32,
     sleep_queue: u64,
     base_url: String,
+    disable_nuser: bool,
+    disable_nsjail: bool,
     tx: tokio::sync::broadcast::Sender<()>,
 ) -> anyhow::Result<()> {
     let instance_name = rd_string(5);
@@ -304,6 +249,8 @@ pub async fn run_workers(
                 &ip,
                 sleep_queue,
                 &base_url,
+                disable_nuser,
+                disable_nsjail,
                 tx,
             )
             .await

backend/src/main.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -66,7 +67,20 @@ async fn main() -> anyhow::Result<()> {
                     .ok()
                     .and_then(|x| x.parse::<u64>().ok())
                     .unwrap_or(windmill::DEFAULT_SLEEP_QUEUE);
+                let disable_nuser = std::env::var("DISABLE_NUSER")
+                    .ok()
+                    .and_then(|x| x.parse::<bool>().ok())
+                    .unwrap_or(false);
+                let disable_nsjail = std::env::var("DISABLE_NSJAIL")
+                    .ok()
+                    .and_then(|x| x.parse::<bool>().ok())
+                    .unwrap_or(false);
 
+                tracing::info!(
+                    "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
+                     {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
+                     {timeout}"
+                );
                 windmill::run_workers(
                     db.clone(),
                     addr,
@@ -74,6 +88,8 @@ async fn main() -> anyhow::Result<()> {
                     num_workers,
                     sleep_queue,
                     base_url,
+                    disable_nuser,
+                    disable_nsjail,
                     tx.clone(),
                 )
                 .await?;

backend/src/parser.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -25,7 +26,17 @@ pub struct MainArgSignature {
     pub args: Vec<Arg>,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Clone)]
+#[serde(rename_all(serialize = "lowercase"))]
+pub enum InnerTyp {
+    Str,
+    Int,
+    Float,
+    Bytes,
+    Email,
+}
+
+#[derive(Serialize, Clone)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum Typ {
     Str,
@@ -33,13 +44,16 @@ pub enum Typ {
     Float,
     Bool,
     Dict,
-    List,
+    List(InnerTyp),
     Bytes,
     Datetime,
+    Resource(String),
+    Email,
+    Sql,
     Unknown,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Clone)]
 pub struct Arg {
     pub name: String,
     pub typ: Typ,
@@ -47,7 +61,7 @@ pub struct Arg {
     pub has_default: bool,
 }
 
-pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
+pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
     let ast = parser::parse_program(code)
         .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
         .statements;
@@ -85,21 +99,20 @@ pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
                     Arg {
                         name: x.arg,
                         typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
-                            Located {
-                                location: _,
-                                node: ExpressionType::Identifier { name },
-                            } => match name.as_ref() {
-                                "str" => Typ::Str,
-                                "float" => Typ::Float,
-                                "int" => Typ::Int,
-                                "bool" => Typ::Bool,
-                                "dict" => Typ::Dict,
-                                "list" => Typ::List,
-                                "bytes" => Typ::Bytes,
-                                "datetime" => Typ::Datetime,
-                                "datetime.datetime" => Typ::Datetime,
-                                _ => Typ::Unknown,
-                            },
+                            Located { location: _, node: ExpressionType::Identifier { name } } => {
+                                match name.as_ref() {
+                                    "str" => Typ::Str,
+                                    "float" => Typ::Float,
+                                    "int" => Typ::Int,
+                                    "bool" => Typ::Bool,
+                                    "dict" => Typ::Dict,
+                                    "list" => Typ::List(InnerTyp::Str),
+                                    "bytes" => Typ::Bytes,
+                                    "datetime" => Typ::Datetime,
+                                    "datetime.datetime" => Typ::Datetime,
+                                    _ => Typ::Unknown,
+                                }
+                            }
                             _ => Typ::Unknown,
                         }),
                         has_default: default.is_some(),
@@ -115,6 +128,165 @@ pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
     }
 }
 
+use swc_common::{sync::Lrc, FileName, SourceMap};
+use swc_ecma_ast::{
+    AssignPat, BindingIdent, Decl, ExportDecl, FnDecl, Ident, ModuleDecl, ModuleItem, Pat,
+    TsArrayType, TsEntityName, TsKeywordTypeKind, TsType, TsTypeRef,
+};
+use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+
+pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Typescript(TsConfig::default()),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let ast = parser
+        .parse_module()
+        .map_err(|e| {
+            error::Error::ExecutionErr(format!("impossible to parse module: {err_s}\n{e:?}"))
+        })?
+        .body;
+
+    // println!("{ast:?}");
+    let params =
+        ast.into_iter().find_map(|x| match x {
+            ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
+                decl:
+                    Decl::Fn(FnDecl {
+                        ident: Ident { span: _, sym, optional: _ },
+                        declare: _,
+                        function,
+                    }),
+                span: _,
+            })) if &sym.to_string() == "main" => Some(function.params),
+            _ => None,
+        });
+    if let Some(params) = params {
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: params
+                .into_iter()
+                .map(|x| match x.pat {
+                    Pat::Ident(ident) => {
+                        let (name, typ) = binding_ident_to_arg(&ident)?;
+                        Ok(Arg { name, typ, default: None, has_default: ident.id.optional })
+                    }
+                    Pat::Assign(AssignPat { span: _, left, right, type_ann: _ }) => {
+                        let (name, typ) =
+                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
+                                error::Error::ExecutionErr(format!(
+                                    "Arg {left:?} has unexpected syntax"
+                                ))
+                            })??;
+                        Ok(Arg {
+                            name,
+                            typ,
+                            default: serde_json::to_value(right)
+                                .map_err(|e| error::Error::ExecutionErr(e.to_string()))?
+                                .as_object()
+                                .and_then(|x| x.get("value").to_owned())
+                                .cloned(),
+
+                            has_default: true,
+                        })
+                    }
+                    _ => Err(error::Error::ExecutionErr(format!(
+                        "Arg {x:?} has unexpected syntax"
+                    ))),
+                })
+                .collect::<Result<Vec<Arg>, error::Error>>()?,
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable (expected to find 'export main function(...)'"
+                .to_string(),
+        ))
+    }
+}
+
+fn binding_ident_to_arg(
+    BindingIdent { id, type_ann }: &BindingIdent,
+) -> anyhow::Result<(String, Typ)> {
+    Ok((
+        id.sym.to_string(),
+        type_ann
+            .as_ref()
+            .map(|x| {
+                match &*x.type_ann {
+                    TsType::TsKeywordType(t) => match t.kind {
+                        TsKeywordTypeKind::TsObjectKeyword => Typ::Dict,
+                        TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
+                        TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
+                        TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
+                        TsKeywordTypeKind::TsStringKeyword => Typ::Str,
+                        _ => Typ::Unknown,
+                    },
+                    // TODO: we can do better here and extract the inner type of array
+                    TsType::TsArrayType(TsArrayType { span: _, elem_type }) => {
+                        match &**elem_type {
+                            TsType::TsTypeRef(TsTypeRef {
+                                span: _,
+                                type_name: TsEntityName::Ident(Ident { span: _, sym, optional: _ }),
+                                type_params: _,
+                            }) => match sym.to_string().as_str() {
+                                "Base64" => Typ::List(InnerTyp::Bytes),
+                                "Email" => Typ::List(InnerTyp::Email),
+                                "bigint" => Typ::List(InnerTyp::Int),
+                                "number" => Typ::List(InnerTyp::Float),
+                                _ => Typ::List(InnerTyp::Str),
+                            },
+                            //TsType::TsKeywordType(())
+                            _ => Typ::List(InnerTyp::Str),
+                        }
+                    }
+                    TsType::TsTypeRef(TsTypeRef { span: _, type_name, type_params }) => {
+                        let sym = match type_name {
+                            TsEntityName::Ident(Ident { span: _, sym, optional: _ }) => sym,
+                            TsEntityName::TsQualifiedName(p) => &*p.right.sym,
+                        };
+                        match sym.to_string().as_str() {
+                            "Resource" => Typ::Resource(
+                                type_params
+                                    .as_ref()
+                                    .and_then(|x| {
+                                        x.params.get(0).and_then(|y| {
+                                            y.as_ts_lit_type().and_then(|z| {
+                                                z.lit
+                                                    .as_str()
+                                                    .map(|a| a.to_owned().value.to_string())
+                                            })
+                                        })
+                                    })
+                                    .unwrap_or_else(|| "unknown".to_string()),
+                            ),
+                            "Base64" => Typ::Bytes,
+                            "Email" => Typ::Email,
+                            "Sql" => Typ::Sql,
+                            _ => Typ::Unknown,
+                        }
+                    }
+                    _ => Typ::Unknown,
+                }
+            })
+            .unwrap_or(Typ::Unknown),
+    ))
+}
+
 const STDIMPORTS: [&str; 301] = [
     "__future__",
     "_abc",
@@ -421,9 +593,7 @@ const STDIMPORTS: [&str; 301] = [
 
 fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
     match et {
-        ExpressionType::String {
-            value: StringGroup::Constant { value },
-        } => Some(json!(value)),
+        ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
         ExpressionType::Number { value } => match value {
             Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
             Number::Float { value } => Some(json!(value)),
@@ -458,17 +628,15 @@ fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
         }
         ExpressionType::None => Some(json!(null)),
 
-        ExpressionType::Call {
-            function: _,
-            args: _,
-            keywords: _,
-        } => Some(json!("<function call>")),
+        ExpressionType::Call { function: _, args: _, keywords: _ } => {
+            Some(json!("<function call>"))
+        }
 
         _ => None,
     }
 }
 
-pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
+pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
     let find_requirements = code
         .lines()
         .find_position(|x| x.starts_with("#requirements:"));
@@ -499,16 +667,14 @@ pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
                             .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
                             .collect::<Vec<String>>(),
                     ),
-                    StatementType::ImportFrom {
-                        level: _,
-                        module: Some(mod_),
-                        names: _,
-                    } => Some(vec![mod_
-                        .split('.')
-                        .next()
-                        .unwrap_or("")
-                        .to_string()
-                        .replace("_", "-")]),
+                    StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
+                        Some(vec![mod_
+                            .split('.')
+                            .next()
+                            .unwrap_or("")
+                            .to_string()
+                            .replace("_", "-")])
+                    }
                     _ => None,
                 },
             })
@@ -527,7 +693,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_parse_sig() -> anyhow::Result<()> {
+    fn test_parse_python_sig() -> anyhow::Result<()> {
         //let code = "print(2 + 3, fd=sys.stderr)";
         let code = "
 
@@ -540,13 +706,13 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
 	return {\"len\": len(name), \"splitted\": name.split() }
 
 ";
-        println!("{}", serde_json::to_string(&parse_signature(code)?)?);
+        println!("{}", serde_json::to_string(&parse_python_signature(code)?)?);
 
         Ok(())
     }
 
     #[test]
-    fn test_parse_imports() -> anyhow::Result<()> {
+    fn test_parse_python_imports() -> anyhow::Result<()> {
         //let code = "print(2 + 3, fd=sys.stderr)";
         let code = "
 
@@ -559,14 +725,14 @@ def main():
     pass
 
 ";
-        let r = parse_imports(code)?;
+        let r = parse_python_imports(code)?;
         println!("{}", serde_json::to_string(&r)?);
         assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
         Ok(())
     }
 
     #[test]
-    fn test_parse_imports2() -> anyhow::Result<()> {
+    fn test_parse_python_imports2() -> anyhow::Result<()> {
         //let code = "print(2 + 3, fd=sys.stderr)";
         let code = "
 #requirements:
@@ -583,10 +749,25 @@ def main():
     pass
 
 ";
-        let r = parse_imports(code)?;
+        let r = parse_python_imports(code)?;
         println!("{}", serde_json::to_string(&r)?);
         assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
 
         Ok(())
     }
+
+    #[test]
+    fn test_parse_deno_sig() -> anyhow::Result<()> {
+        let code = "
+
+export function main(test1?: string, test2: string = \"burkina\",
+    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], email: Email) {
+    console.log(42)
+}
+
+";
+        println!("{}", serde_json::to_string(&parse_deno_signature(code)?)?);
+
+        Ok(())
+    }
 }

backend/src/resources.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -26,6 +27,7 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
         .route("/get/*path", get(get_resource))
+        .route("/exists/*path", get(exists_resource))
         .route("/get_value/*path", get(get_resource_value))
         .route("/update/*path", post(update_resource))
         .route("/delete/*path", delete(delete_resource))
@@ -33,6 +35,7 @@ pub fn workspaced_service() -> Router {
         .route("/type/list", get(list_resource_types))
         .route("/type/listnames", get(list_resource_types_names))
         .route("/type/get/:name", get(get_resource_type))
+        .route("/type/exists/:name", get(exists_resource_type))
         .route("/type/update/:name", post(update_resource_type))
         .route("/type/delete/:name", delete(delete_resource_type))
         .route("/type/create", post(create_resource_type))
@@ -67,6 +70,7 @@ pub struct Resource {
     pub description: Option<String>,
     pub resource_type: String,
     pub extra_perms: serde_json::Value,
+    pub is_oauth: bool,
 }
 
 #[derive(Deserialize)]
@@ -75,6 +79,7 @@ pub struct CreateResource {
     pub value: Option<serde_json::Value>,
     pub description: Option<String>,
     pub resource_type: String,
+    pub is_oauth: Option<bool>,
 }
 #[derive(Deserialize)]
 struct EditResource {
@@ -104,6 +109,7 @@ async fn list_resources(
             "description",
             "resource_type",
             "extra_perms",
+            "is_oauth",
         ])
         .order_by("path", true)
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -135,7 +141,8 @@ async fn get_resource(
 
     let resource_o = sqlx::query_as!(
         Resource,
-        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         path.to_owned(),
         &w_id
     )
@@ -147,6 +154,24 @@ async fn get_resource(
     Ok(Json(resource))
 }
 
+async fn exists_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn get_resource_value(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -156,7 +181,8 @@ async fn get_resource_value(
     let mut tx = user_db.begin(&authed).await?;
 
     let value_o = sqlx::query_scalar!(
-        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         path.to_owned(),
         &w_id
     )
@@ -178,13 +204,14 @@ async fn create_resource(
 
     sqlx::query!(
         "INSERT INTO resource
-            (workspace_id, path, value, description, resource_type)
-            VALUES ($1, $2, $3, $4, $5)",
+            (workspace_id, path, value, description, resource_type, is_oauth)
+            VALUES ($1, $2, $3, $4, $5, $6)",
         w_id,
         resource.path,
         resource.value,
         resource.description,
         resource.resource_type,
+        resource.is_oauth.unwrap_or(false)
     )
     .execute(&mut tx)
     .await?;
@@ -259,10 +286,16 @@ async fn update_resource(
     if let Some(ndesc) = ns.description {
         sqlb.set_str("description", ndesc);
     }
+
+    sqlb.returning("path");
+
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
-    sqlx::query(&sql).execute(&mut tx).await?;
+    let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+
+    let npath = crate::utils::not_found_if_none(npath_o, "Resource", path)?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -275,16 +308,21 @@ async fn update_resource(
     .await?;
     tx.commit().await?;
 
-    Ok(format!("resource {} updated (npath: {:?})", path, ns.path))
+    Ok(format!("resource {} updated (npath: {:?})", path, npath))
 }
 
 async fn list_resource_types(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<ResourceType>> {
-    let rows = sqlx::query_as!(ResourceType, "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
-        .fetch_all(&db)
-        .await?;
+    let rows = sqlx::query_as!(
+        ResourceType,
+        "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER \
+         BY name",
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
 
     Ok(Json(rows))
 }
@@ -293,9 +331,13 @@ async fn list_resource_types_names(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
-    let rows = sqlx::query_scalar!("SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
-        .fetch_all(&db)
-        .await?;
+    let rows = sqlx::query_scalar!(
+        "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') \
+         ORDER BY name",
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
 
     Ok(Json(rows))
 }
@@ -309,7 +351,8 @@ async fn get_resource_type(
 
     let resource_type_o = sqlx::query_as!(
         ResourceType,
-        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+        "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = \
+         'starter')",
         &name,
         &w_id
     )
@@ -321,6 +364,22 @@ async fn get_resource_type(
     Ok(Json(resource_type))
 }
 
+async fn exists_resource_type(
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<bool> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn create_resource_type(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/src/schedule.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -9,15 +10,15 @@ use std::str::FromStr;
 
 use crate::{
     audit::{audit_log, ActionKind},
-    db::UserDB,
-    error::{self, JsonResult, Result},
+    db::{UserDB, DB},
+    error::{self, Error, JsonResult, Result},
     jobs::{self, push, JobPayload},
     users::Authed,
     utils::{get_owner_from_path, Pagination, StripPath},
 };
 use axum::{
     extract::{Extension, Path, Query},
-    routing::{get, post},
+    routing::{delete, get, post},
     Json, Router,
 };
 
@@ -30,8 +31,10 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_schedule))
         .route("/get/*path", get(get_schedule))
+        .route("/exists/*path", get(exists_schedule))
         .route("/create", post(create_schedule))
         .route("/update/*path", post(edit_schedule))
+        .route("/delete/*path", delete(delete_schedule))
         .route("/setenabled/*path", post(set_enabled))
 }
 
@@ -62,6 +65,7 @@ pub struct NewSchedule {
     pub script_path: String,
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
+    pub enabled: Option<bool>,
 }
 
 pub async fn push_scheduled_job<'c>(
@@ -128,8 +132,12 @@ async fn create_schedule(
     cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    let schedule = sqlx::query_as!(Schedule,
-        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, is_flow, args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",
+    check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
+
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
+         is_flow, args, enabled) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
@@ -137,10 +145,12 @@ async fn create_schedule(
         &authed.username,
         ns.script_path,
         ns.is_flow,
-        ns.args
+        ns.args,
+        ns.enabled
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
 
     audit_log(
         &mut tx,
@@ -161,11 +171,43 @@ async fn create_schedule(
     )
     .await?;
 
-    let tx = push_scheduled_job(tx, schedule).await?;
+    let tx = if ns.enabled.unwrap_or(true) {
+        push_scheduled_job(tx, schedule).await?
+    } else {
+        tx
+    };
     tx.commit().await?;
+
     Ok(ns.path.to_string())
 }
 
+async fn check_flow_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+    is_flow: bool,
+    script_path: &str,
+) -> error::Result<()> {
+    if path != script_path || !is_flow {
+        let exists_flow = sqlx::query_scalar!(
+            "SELECT EXISTS (SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
+            path,
+            w_id
+        )
+        .fetch_one(tx)
+        .await?
+        .unwrap_or(false);
+        if exists_flow {
+            return Err(error::Error::BadConfig(format!(
+                "If a schedule has the same path as or a flow, it must be its primary schedule \
+                 and hence can only trigger it. 
+            However the provided path is: {script_path} and is_flow is: {is_flow}",
+            )));
+        };
+    }
+    Ok(())
+}
+
 #[derive(Deserialize)]
 pub struct EditSchedule {
     pub schedule: String,
@@ -193,9 +235,13 @@ async fn edit_schedule(
 
     let mut tx = user_db.begin(&authed).await?;
 
+    check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?;
+
     clear_schedule(&mut tx, path).await?;
-    let schedule = sqlx::query_as!(Schedule,
-        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
+    let schedule = sqlx::query_as!(
+        Schedule,
+        "UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path \
+         = $5 AND workspace_id = $6 RETURNING *",
         es.schedule,
         es.script_path,
         es.is_flow,
@@ -204,7 +250,8 @@ async fn edit_schedule(
         w_id,
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
 
     if schedule.enabled {
         tx = push_scheduled_job(tx, schedule).await?;
@@ -284,6 +331,24 @@ async fn get_schedule(
     Ok(Json(schedule))
 }
 
+async fn exists_schedule(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 #[derive(Deserialize)]
 pub struct PreviewPayload {
     pub schedule: String,
@@ -355,6 +420,38 @@ pub async fn set_enabled(
     ))
 }
 
+async fn delete_schedule(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "schedule.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(format!("schedule {} deleted", path))
+}
+
 fn schedule_to_user(path: &str) -> String {
     format!("schedule-{}", path.replace('/', "-"))
 }

backend/src/scripts.rs

@@ -1,23 +1,25 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
+use reqwest::Client;
 use serde::Deserializer;
 use sql_builder::prelude::*;
 
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{Error, JsonResult, Result},
+    error::{to_anyhow, Error, JsonResult, Result},
     jobs, parser,
     users::{owner_to_token_owner, truncate_token, Authed, Tokened},
-    utils::{require_admin, Pagination, StripPath},
+    utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
 };
 use axum::{
-    extract::{Extension, Path, Query},
+    extract::{Extension, Host, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -35,7 +37,14 @@ use std::{
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
 pub fn global_service() -> Router {
-    Router::new().route("/tojsonschema", post(parse_code_to_jsonschema))
+    Router::new()
+        .route(
+            "/python/tojsonschema",
+            post(parse_python_code_to_jsonschema),
+        )
+        .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema))
+        .route("/hub/list", get(list_hub_scripts))
+        .route("/hub/get/*path", get(get_hub_script_by_path))
 }
 
 pub fn workspaced_service() -> Router {
@@ -44,12 +53,20 @@ pub fn workspaced_service() -> Router {
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
         .route("/get/p/*path", get(get_script_by_path))
+        .route("/exists/p/*path", get(exists_script_by_path))
         .route("/archive/h/:hash", post(archive_script_by_hash))
         .route("/delete/h/:hash", post(delete_script_by_hash))
         .route("/get/h/:hash", get(get_script_by_hash))
         .route("/deployment_status/h/:hash", get(get_deployment_status))
 }
 
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
+#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum ScriptLang {
+    Deno,
+    Python3,
+}
 #[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
@@ -113,6 +130,8 @@ pub struct Script {
     pub extra_perms: serde_json::Value,
     pub lock: Option<String>,
     pub lock_error_logs: Option<String>,
+    pub language: ScriptLang,
+    pub is_trigger: bool,
 }
 
 #[derive(Serialize, Deserialize, sqlx::Type, Debug)]
@@ -138,6 +157,8 @@ pub struct NewScript {
     pub schema: Option<Schema>,
     pub is_template: Option<bool>,
     pub lock: Option<Vec<String>>,
+    pub language: ScriptLang,
+    pub is_trigger: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -152,6 +173,7 @@ pub struct ListScriptQuery {
     pub order_by: Option<String>,
     pub order_desc: Option<bool>,
     pub is_template: Option<bool>,
+    pub is_trigger: Option<bool>,
 }
 
 async fn list_scripts(
@@ -175,12 +197,14 @@ async fn list_scripts(
             "created_by",
             "created_at",
             "archived",
-            "schema",
+            "null as schema",
             "deleted",
             "is_template",
             "extra_perms",
             "null as lock",
             "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
+            "language",
+            "is_trigger",
         ])
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -218,6 +242,9 @@ async fn list_scripts(
     if let Some(it) = &lq.is_template {
         sqlb.and_where_eq("is_template", it);
     }
+    if let Some(it) = &lq.is_trigger {
+        sqlb.and_where_eq("is_trigger", it);
+    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
@@ -226,6 +253,22 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
+async fn list_hub_scripts(
+    Authed { email, username, .. }: Authed,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> JsonResult<serde_json::Value> {
+    let asks = list_elems_from_hub(
+        http_client,
+        "https://hub.windmill.dev/searchData?approved=true",
+        email,
+        username,
+        host,
+    )
+    .await?;
+    Ok(Json(asks))
+}
+
 fn hash_script(ns: &NewScript) -> i64 {
     let mut dh = DefaultHasher::new();
     ns.hash(&mut dh);
@@ -298,7 +341,7 @@ async fn create_script(
                 if let Some(clashing_hash) = clashing_hash_o {
                     return Err(Error::BadRequest(format!(
                         "A script with hash {} with same parent_hash has been found. However, the \
-                     lineage must be linear: no 2 scripts can have the same parent",
+                         lineage must be linear: no 2 scripts can have the same parent",
                         ScriptHash(clashing_hash)
                     )));
                 };
@@ -344,10 +387,16 @@ async fn create_script(
         .map(|v| v.1.clone())
         .unwrap_or(json!({}));
 
+    let lock = if ns.language == ScriptLang::Deno {
+        Some("".to_string())
+    } else {
+        ns.lock.as_ref().map(|x| x.join("\n"))
+    };
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
-        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \
-         created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
+        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
+         content, created_by, schema, is_template, extra_perms, lock, language, is_trigger) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
         &w_id,
         &hash.0,
         ns.path,
@@ -359,13 +408,15 @@ async fn create_script(
         ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
         ns.is_template.unwrap_or(false),
         extra_perms,
-        ns.lock.as_ref().map(|x| x.join("\n"))
+        lock,
+        ns.language: ScriptLang,
+        ns.is_trigger.unwrap_or(false),
     )
     .execute(&mut tx)
     .await?;
 
-    let mut tx = if ns.lock.is_none() {
-        let dependencies = parser::parse_imports(&ns.content)?;
+    let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
+        let dependencies = parser::parse_python_imports(&ns.content)?;
         let (_, tx) = jobs::push(
             tx,
             &w_id,
@@ -426,6 +477,32 @@ async fn create_script(
     Ok((StatusCode::CREATED, format!("{}", hash)))
 }
 
+pub async fn get_hub_script_by_path(
+    Authed { email, username, .. }: Authed,
+    Path(path): Path<StripPath>,
+    Extension(http_client): Extension<Client>,
+    Host(host): Host,
+) -> Result<String> {
+    let path = path
+        .to_path()
+        .strip_prefix("hub/")
+        .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
+
+    let content = http_get_from_hub(
+        http_client,
+        &format!("https://hub.windmill.dev/raw/{path}.ts"),
+        email,
+        username,
+        host,
+        true,
+    )
+    .await?
+    .text()
+    .await
+    .map_err(to_anyhow)?;
+    Ok(content)
+}
+
 async fn get_script_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -435,8 +512,10 @@ async fn get_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND (workspace_id = $2 OR workspace_id = 'starter'))",
+        "SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') \
+         AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
+         (workspace_id = $2 OR workspace_id = 'starter'))",
     )
     .bind(path)
     .bind(w_id)
@@ -448,6 +527,27 @@ async fn get_script_by_path(
     Ok(Json(script))
 }
 
+async fn exists_script_by_path(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND (workspace_id = $2 OR \
+         workspace_id = 'starter') AND
+         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 \
+         OR workspace_id = 'starter')))",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn get_script_by_hash_internal<'c>(
     db: &mut Transaction<'c, Postgres>,
     workspace_id: &str,
@@ -488,8 +588,10 @@ async fn get_deployment_status(
     Path((w_id, hash)): Path<(String, ScriptHash)>,
 ) -> JsonResult<DeploymentStatus> {
     let mut tx = user_db.begin(&authed).await?;
-    let status_o: Option<DeploymentStatus> = sqlx::query_as!(DeploymentStatus,
-        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
+    let status_o: Option<DeploymentStatus> = sqlx::query_as!(
+        DeploymentStatus,
+        "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR \
+         workspace_id = 'starter')",
         hash.0,
         w_id,
     )
@@ -517,7 +619,8 @@ async fn archive_script_by_path(
         &w_id
     )
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -545,7 +648,9 @@ async fn archive_script_by_hash(
     )
     .bind(&hash.0)
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -571,13 +676,15 @@ async fn delete_script_by_hash(
 
     require_admin(authed.is_admin, &authed.username)?;
     let script = sqlx::query_as::<_, Script>(
-        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND workspace_id = $2\
-         RETURNING *",
+        "UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND \
+         workspace_id = $2RETURNING *",
     )
     .bind(&hash.0)
     .bind(&w_id)
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("deleting script by hash {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -593,10 +700,16 @@ async fn delete_script_by_hash(
     Ok(Json(script))
 }
 
-async fn parse_code_to_jsonschema(
+async fn parse_python_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<parser::MainArgSignature> {
-    parser::parse_signature(&code).map(Json)
+    parser::parse_python_signature(&code).map(Json)
+}
+
+async fn parse_deno_code_to_jsonschema(
+    Json(code): Json<String>,
+) -> JsonResult<parser::MainArgSignature> {
+    parser::parse_deno_signature(&code).map(Json)
 }
 
 pub fn to_i64(s: &str) -> Result<i64> {

backend/src/static_assets.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.

backend/src/tracing_init.rs

@@ -0,0 +1,97 @@
+use ::tracing::{field, Metadata, Span};
+use ::tracing_subscriber::{
+    filter::filter_fn,
+    fmt::{format, Layer},
+    prelude::*,
+    EnvFilter,
+};
+use hyper::Response;
+use tower_http::trace::{MakeSpan, OnResponse};
+
+#[derive(Clone)]
+pub struct MyOnResponse {}
+
+impl<B> OnResponse<B> for MyOnResponse {
+    fn on_response(
+        self,
+        response: &Response<B>,
+        latency: std::time::Duration,
+        _span: &tracing::Span,
+    ) {
+        tracing::info!(
+            latency = latency.as_millis(),
+            status = response.status().as_u16(),
+            "response"
+        )
+    }
+}
+
+#[derive(Clone)]
+pub struct MyMakeSpan {}
+
+impl<B> MakeSpan<B> for MyMakeSpan {
+    fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
+        tracing::info_span!(
+            "request",
+            method = %request.method(),
+            uri = %request.uri(),
+            username = field::Empty,
+            workspace_id = field::Empty,
+            email = field::Empty,
+        )
+    }
+}
+
+fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
+    tracing_subscriber::fmt::layer()
+        .json()
+        .flatten_event(true)
+        .with_span_list(false)
+        .with_current_span(true)
+}
+
+fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::Compact>> {
+    tracing_subscriber::fmt::layer().compact()
+}
+
+fn filter_metadata(meta: &Metadata) -> bool {
+    meta.target().starts_with("windmill")
+}
+
+pub async fn initialize_tracing() -> anyhow::Result<()> {
+    let tokio_console = std::env::var("TOKIO_CONSOLE")
+        .map(|x| x == "true")
+        .unwrap_or(false);
+    let json_fmt = std::env::var("JSON_FMT")
+        .map(|x| x == "true")
+        .unwrap_or(false);
+
+    let env_filter = EnvFilter::from_default_env();
+
+    let nenv_filter = if tokio_console {
+        env_filter
+            .add_directive("runtime=trace".parse()?)
+            .add_directive("tokio=trace".parse()?)
+    } else {
+        env_filter
+    };
+    let ts_base = tracing_subscriber::registry().with(nenv_filter);
+
+    match (json_fmt, tokio_console) {
+        (true, true) => ts_base
+            .with(json_layer().with_filter(filter_fn(filter_metadata)))
+            .with(console_subscriber::spawn())
+            .init(),
+        (true, false) => ts_base
+            .with(json_layer().with_filter(filter_fn(filter_metadata)))
+            .init(),
+        (false, true) => ts_base
+            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
+            .with(console_subscriber::spawn())
+            .init(),
+        _ => ts_base
+            .with(compact_layer().with_filter(filter_fn(filter_metadata)))
+            .init(),
+    }
+    Ok(())
+}

backend/src/users.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -10,13 +11,13 @@ use std::{sync::Arc, time::Duration};
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{Error, JsonResult, Result, self},
-    utils::{require_admin, require_super_admin, Pagination}
+    error::{self, Error, JsonResult, Result},
+    utils::{require_admin, require_super_admin, Pagination},
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
     async_trait,
-    extract::{Extension, FromRequest, Path, RequestParts, Query},
+    extract::{Extension, FromRequest, Path, Query, RequestParts},
     http,
     routing::{delete, get, post},
     Json, Router,
@@ -25,7 +26,7 @@ use hyper::StatusCode;
 use rand::rngs::OsRng;
 use retainer::Cache;
 use serde::{Deserialize, Serialize};
-use sqlx::{FromRow};
+use sqlx::FromRow;
 use time::OffsetDateTime;
 use tower_cookies::{Cookie, Cookies};
 use tracing::Span;
@@ -48,8 +49,6 @@ pub fn workspaced_service() -> Router {
         .route("/leave", post(leave_workspace))
 }
 
-
-
 pub fn global_service() -> Router {
     Router::new()
         .route("/email", get(get_email))
@@ -59,18 +58,17 @@ pub fn global_service() -> Router {
         .route("/accept_invite", post(accept_invite))
         .route("/list_as_super_admin", get(list_users_as_super_admin))
         .route("/setpassword", post(set_password))
+        .route("/create", post(create_user))
         .route("/update/:user", post(update_user))
         .route("/logout", post(logout))
         .route("/tokens/create", post(create_token))
         .route("/tokens/delete/:token_prefix", delete(delete_token))
         .route("/tokens/list", get(list_tokens))
-        // .route("/list_invite_codes", get(list_invite_codes))
-        // .route("/create_invite_code", post(create_invite_code))
-        // .route("/signup", post(signup))
-        // .route("/lost_password", post(lost_password))
-        // .route("/use_magic_link", get(use_magic_link))
-
-
+    // .route("/list_invite_codes", get(list_invite_codes))
+    // .route("/create_invite_code", post(create_invite_code))
+    // .route("/signup", post(signup))
+    // .route("/lost_password", post(lost_password))
+    // .route("/use_magic_link", get(use_magic_link))
 }
 
 pub fn make_unauthed_service() -> Router {
@@ -84,22 +82,21 @@ pub struct AuthCache {
 
 impl AuthCache {
     pub fn new(db: DB) -> Self {
-        AuthCache {
-            cache: Cache::new(),
-            db,
-        }
+        AuthCache { cache: Cache::new(), db }
     }
 
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
-        let key = (w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string());
+        let key = (
+            w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
+            token.to_string(),
+        );
         let s = self.cache.get(&key).await.map(|c| c.to_owned());
         match s {
-            a @ Some(_) => {
-                a
-            },
+            a @ Some(_) => a,
             None => {
                 let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool)>(
-                    "UPDATE token SET last_used_at = $1 WHERE token = $2 AND (expiration > NOW() OR expiration IS NULL) RETURNING owner, email, super_admin",
+                    "UPDATE token SET last_used_at = $1 WHERE token = $2 AND (expiration > NOW() \
+                     OR expiration IS NULL) RETURNING owner, email, super_admin",
                 )
                 .bind(chrono::Utc::now())
                 .bind(token)
@@ -113,40 +110,44 @@ impl AuthCache {
                         match user {
                             (_, Some(email), super_admin) => {
                                 if w_id.is_some() {
-                                    let row_o = 
-                                    sqlx::query_as::<_, (String, bool)>(
-                                "SELECT username, is_admin FROM usr where email = $1 AND workspace_id = $2",
-                            )
-                            .bind(&email)
-                            .bind(&w_id.as_ref().unwrap())
-                            .fetch_optional(&self.db)
-                            .await
-                            .unwrap_or(Some(("error".to_string(), false)));
+                                    let row_o = sqlx::query_as::<_, (String, bool)>(
+                                        "SELECT username, is_admin FROM usr where email = $1 AND \
+                                         workspace_id = $2",
+                                    )
+                                    .bind(&email)
+                                    .bind(&w_id.as_ref().unwrap())
+                                    .fetch_optional(&self.db)
+                                    .await
+                                    .unwrap_or(Some(("error".to_string(), false)));
 
-                            match row_o {
-                                Some((username, is_admin)) => {
-                             let groups = get_groups_for_user(&w_id.as_ref().unwrap(),
-                                 &username, &self.db)
-                                .await
-                                .ok()
-                                .unwrap_or_default();
+                                    match row_o {
+                                        Some((username, is_admin)) => {
+                                            let groups = get_groups_for_user(
+                                                &w_id.as_ref().unwrap(),
+                                                &username,
+                                                &self.db,
+                                            )
+                                            .await
+                                            .ok()
+                                            .unwrap_or_default();
 
-                                    Some(Authed {
-                                        email: Some(email),
-                                        username,
-                                        is_admin: is_admin || super_admin,
-                                        groups,
-                                    })
-                                }, 
-                                None if super_admin || w_id.unwrap() == "starter" => Some(Authed {
-                                        email: Some(email.to_string()),
-                                        username: email,
-                                        is_admin: super_admin,
-                                        groups: vec![],
-                                    }),
-                                None => None
-                            
-                            }
+                                            Some(Authed {
+                                                email: Some(email),
+                                                username,
+                                                is_admin: is_admin || super_admin,
+                                                groups,
+                                            })
+                                        }
+                                        None if super_admin || w_id.unwrap() == "starter" => {
+                                            Some(Authed {
+                                                email: Some(email.to_string()),
+                                                username: email,
+                                                is_admin: super_admin,
+                                                groups: vec![],
+                                            })
+                                        }
+                                        None => None,
+                                    }
                                 } else {
                                     Some(Authed {
                                         email: Some(email.to_string()),
@@ -159,21 +160,23 @@ impl AuthCache {
                             (Some(owner), _, super_admin) if w_id.is_some() => {
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
-
-                                        let is_admin = super_admin || sqlx::query_scalar!(
-                                            "SELECT is_admin FROM usr where username = $1 AND workspace_id = $2",
-                                            name,
-                                            &w_id.as_ref().unwrap()
-                                        )
-                                        .fetch_one(&self.db)
-                                        .await
-                                        .ok()
-                                        .unwrap_or(false);
-
-                                        let groups = get_groups_for_user(&w_id.unwrap(), &name, &self.db)
+                                        let is_admin = super_admin
+                                            || sqlx::query_scalar!(
+                                                "SELECT is_admin FROM usr where username = $1 AND \
+                                                 workspace_id = $2",
+                                                name,
+                                                &w_id.as_ref().unwrap()
+                                            )
+                                            .fetch_one(&self.db)
                                             .await
                                             .ok()
-                                            .unwrap_or_default();
+                                            .unwrap_or(false);
+
+                                        let groups =
+                                            get_groups_for_user(&w_id.unwrap(), &name, &self.db)
+                                                .await
+                                                .ok()
+                                                .unwrap_or_default();
 
                                         Some(Authed {
                                             email: None,
@@ -216,7 +219,8 @@ impl AuthCache {
 
 async fn extract_token<B: Send>(req: &mut RequestParts<B>) -> Option<String> {
     let auth_header = req
-        .headers().get(http::header::AUTHORIZATION)
+        .headers()
+        .get(http::header::AUTHORIZATION)
         .and_then(|value| value.to_str().ok())
         .and_then(|s| s.strip_prefix("Bearer "));
 
@@ -247,14 +251,14 @@ where
             Ok(tokened.clone())
         } else {
             let token_o = extract_token(req).await;
-                if let Some(token) = token_o {
-                    let tokened = Self { token };
-                    req.extensions_mut().insert(tokened.clone());
-                    Ok(tokened)
-                } else {
-                    Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
-                }
+            if let Some(token) = token_o {
+                let tokened = Self { token };
+                req.extensions_mut().insert(tokened.clone());
+                Ok(tokened)
+            } else {
+                Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
             }
+        }
     }
 }
 
@@ -292,9 +296,15 @@ where
             };
             if let Some(token) = token_o {
                 if let Ok(Extension(cache)) = Extension::<Arc<AuthCache>>::from_request(req).await {
-                    if let Some(authed) = cache.get_authed(workspace_id, &token).await {
+                    if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
                         req.extensions_mut().insert(authed.clone());
                         Span::current().record("username", &authed.username.as_str());
+                        if let Some(email) = authed.email.clone() {
+                            Span::current().record("email", &email.as_str());
+                        }
+                        if let Some(workspace_id) = workspace_id {
+                            Span::current().record("workspace_id", &workspace_id);
+                        }
                         return Ok(authed);
                     }
                 }
@@ -313,7 +323,31 @@ pub struct User {
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub operator: bool,
     pub disabled: bool,
-    pub role: Option<String>
+    pub role: Option<String>,
+}
+
+#[derive(FromRow, Serialize)]
+pub struct Usage {
+    pub duration_ms: i64,
+    pub jobs: i64,
+    pub flows: i64,
+}
+
+#[derive(Serialize)]
+pub struct UserWithUsage {
+    #[serde(flatten)]
+    pub user: User,
+    pub usage: Usage,
+}
+
+#[derive(FromRow, Serialize)]
+pub struct GlobalUserInfo {
+    email: String,
+    login_type: Option<String>,
+    super_admin: bool,
+    verified: bool,
+    name: Option<String>,
+    company: Option<String>,
 }
 
 #[derive(Serialize)]
@@ -327,7 +361,7 @@ pub struct UserInfo {
     pub groups: Vec<String>,
     pub operator: bool,
     pub disabled: bool,
-    pub role: Option<String>
+    pub role: Option<String>,
 }
 
 #[derive(FromRow, Serialize)]
@@ -368,10 +402,10 @@ pub struct NewUser {
     pub email: String,
     pub password: String,
     pub super_admin: bool,
+    pub name: Option<String>,
+    pub company: Option<String>,
 }
 
-
-
 #[derive(Deserialize)]
 pub struct AcceptInvite {
     pub workspace_id: String,
@@ -385,7 +419,6 @@ pub struct DeclineInvite {
 
 #[derive(Deserialize)]
 pub struct EditUser {
-    pub email: String,
     pub is_super_admin: Option<bool>,
 }
 
@@ -421,7 +454,6 @@ pub struct Login {
     pub password: String,
 }
 
-
 #[derive(Deserialize)]
 pub struct Signup {
     pub email: String,
@@ -440,27 +472,21 @@ struct WorkspaceUsername {
     pub username: String,
 }
 
-#[derive(sqlx::Type, Serialize, Deserialize)]
-#[sqlx(type_name = "LOGIN_TYPE", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase"))]
-pub enum LoginType {
-    Password,
-    Github,
-}
-
 async fn exists_username(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
-    Json(WorkspaceUsername { username }): Json<WorkspaceUsername>
+    Json(WorkspaceUsername { username }): Json<WorkspaceUsername>,
 ) -> JsonResult<bool> {
     let mut tx = user_db.begin(&authed).await?;
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", 
-        &w_id, &username)
-        .fetch_one(&mut tx)
-        .await?
-        .unwrap_or(false);
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)",
+        &w_id,
+        &username
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
     tx.commit().await?;
     Ok(Json(exists))
 }
@@ -468,12 +494,33 @@ async fn exists_username(
 async fn list_users(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>
-) -> JsonResult<Vec<User>> {
+    Path(w_id): Path<String>,
+) -> JsonResult<Vec<UserWithUsage>> {
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as!(User, "SELECT * from usr WHERE workspace_id = $1", &w_id)
-        .fetch_all(&mut tx)
-        .await?;
+    let rows = sqlx::query(
+        "
+        SELECT usr.*, usage.*
+          FROM usr
+             , LATERAL (
+                SELECT COALESCE(SUM(duration_ms), 0)  duration_ms
+                     , COALESCE(SUM(job_kind     IN ('flow', 'flowpreview') ::int), 0)  flows
+                     , COALESCE(SUM(job_kind NOT IN ('flow', 'flowpreview') ::int), 0)  jobs
+                  FROM completed_job
+                 WHERE workspace_id = usr.workspace_id
+                   AND created_by = usr.username
+                   AND parent_job IS NULL
+                   AND now() - '2 week'::interval < created_at 
+               ) usage
+         WHERE workspace_id = $1
+         ",
+    )
+    .bind(&w_id)
+    .try_map(|row| {
+        // flatten not released yet https://github.com/launchbadge/sqlx/pull/1959
+        Ok(UserWithUsage { user: FromRow::from_row(&row)?, usage: FromRow::from_row(&row)? })
+    })
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
@@ -481,20 +528,25 @@ async fn list_users(
 async fn list_users_as_super_admin(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Query(pagination): Query<Pagination>
-) -> JsonResult<Vec<User>> {
+    Query(pagination): Query<Pagination>,
+) -> JsonResult<Vec<GlobalUserInfo>> {
     let mut tx = db.begin().await?;
     require_super_admin(&mut tx, authed.email).await?;
     let (per_page, offset) = crate::utils::paginate(pagination);
 
-    let rows = sqlx::query_as!(User, "SELECT * from usr LIMIT $1 OFFSET $2", per_page as i32, offset as i32)
-        .fetch_all(&mut tx)
-        .await?;
+    let rows = sqlx::query_as!(
+        GlobalUserInfo,
+        "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT \
+         $1 OFFSET $2",
+        per_page as i32,
+        offset as i32
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
-
 // async fn list_invite_codes(
 //     authed: Authed,
 //     Extension(db): Extension<DB>,
@@ -511,11 +563,10 @@ async fn list_users_as_super_admin(
 //     Ok(Json(rows))
 // }
 
-
 async fn list_usernames(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Path(w_id): Path<String>
+    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
     let mut tx = user_db.begin(&authed).await?;
     let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id)
@@ -530,14 +581,17 @@ async fn list_invites(
     Extension(db): Extension<DB>,
 ) -> JsonResult<Vec<WorkspaceInvite>> {
     let mut tx = db.begin().await?;
-    let rows = sqlx::query_as!(WorkspaceInvite, "SELECT * from workspace_invite WHERE email = $1", authed.email)
-        .fetch_all(&mut tx)
-        .await?;
+    let rows = sqlx::query_as!(
+        WorkspaceInvite,
+        "SELECT * from workspace_invite WHERE email = $1",
+        authed.email
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
-
 async fn logout(
     Tokened { token }: Tokened,
     cookies: Cookies,
@@ -588,41 +642,46 @@ async fn whoami(
     }
 }
 
-#[derive(FromRow, Serialize)]
-pub struct GlobalUserInfo {
-    email: String,
-    login_type: Option<String>,
-    super_admin: bool,
-    verified: bool,
-    name: Option<String>,
-    company: Option<String>,
-}
-
 async fn global_whoami(
     Extension(db): Extension<DB>,
     Authed { email, .. }: Authed,
 ) -> JsonResult<GlobalUserInfo> {
-    let user: GlobalUserInfo = sqlx::query_as!(GlobalUserInfo, "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE email = $1", email)
-        .fetch_one(&db)
-        .await?;
+    let user: GlobalUserInfo = sqlx::query_as!(
+        GlobalUserInfo,
+        "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE \
+         email = $1",
+        email
+    )
+    .fetch_one(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")))?;
+
     Ok(Json(user))
 }
 
-async fn get_email(
-    Authed { email, .. }: Authed,
-) -> Result<String> {
-    let email = email.ok_or(Error::BadRequest("current session does not correspond to an user with email".to_string()))?;
+async fn get_email(Authed { email, .. }: Authed) -> Result<String> {
+    let email = email.ok_or(Error::BadRequest(
+        "current session does not correspond to an user with email".to_string(),
+    ))?;
     Ok(email)
 }
 
 async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
-    let user = sqlx::query_as!(User, "SELECT * FROM usr where username = $1 AND workspace_id = $2", username, w_id)
-        .fetch_optional(db)
-        .await?;
-    let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", user.as_ref().map(|x| &x.email))
-        .fetch_optional(db)
-        .await?
-        .unwrap_or(false);
+    let user = sqlx::query_as!(
+        User,
+        "SELECT * FROM usr where username = $1 AND workspace_id = $2",
+        username,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?;
+    let is_super_admin = sqlx::query_scalar!(
+        "SELECT super_admin FROM password WHERE email = $1",
+        user.as_ref().map(|x| &x.email)
+    )
+    .fetch_optional(db)
+    .await?
+    .unwrap_or(false);
     let groups = get_groups_for_user(&w_id, username, db).await?;
     Ok(user.map(|usr| UserInfo {
         groups,
@@ -634,33 +693,36 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
         created_at: usr.created_at,
         operator: usr.operator,
         disabled: usr.disabled,
-        role: usr.role
+        role: usr.role,
     }))
 }
 
 async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<Vec<String>> {
-    let groups = sqlx::query_scalar!("SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2", username, w_id)
-        .fetch_all(db)
-        .await?;
+    let groups = sqlx::query_scalar!(
+        "SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2",
+        username,
+        w_id
+    )
+    .fetch_all(db)
+    .await?;
     Ok(groups)
 }
 
-async fn whois(Extension(db): Extension<DB>, Path((w_id, username)): Path<(String, String)>) -> JsonResult<UserInfo> {
+async fn whois(
+    Extension(db): Extension<DB>,
+    Path((w_id, username)): Path<(String, String)>,
+) -> JsonResult<UserInfo> {
     let user_o = get_user(&w_id, &username, &db).await?;
     let user = crate::utils::not_found_if_none(user_o, "User", username)?;
     Ok(Json(user))
 }
 
-
-
-
 // async fn create_invite_code(
 //     Authed { email, .. }: Authed,
 //     Extension(db): Extension<DB>,
 //     Json(nu): Json<NewInviteCode>,
 // ) -> Result<(StatusCode, String)> {
 
-
 //     let mut tx = db.begin().await?;
 //     require_super_admin(&mut tx, email).await?;
 
@@ -687,7 +749,6 @@ async fn decline_invite(
     Extension(db): Extension<DB>,
     Json(nu): Json<DeclineInvite>,
 ) -> Result<(StatusCode, String)> {
-
     let mut tx = db.begin().await?;
 
     let email = email.unwrap_or("".to_string());
@@ -700,10 +761,10 @@ async fn decline_invite(
     .await?;
 
     audit_log(
-         &mut tx,
-         &email,
+        &mut tx,
+        &email,
         "users.decline_invite",
-        ActionKind::Create,
+        ActionKind::Delete,
         &nu.workspace_id,
         Some(&email),
         None,
@@ -714,7 +775,10 @@ async fn decline_invite(
     if is_admin.is_some() {
         Ok((
             StatusCode::OK,
-            format!("user {} declined invite to workspace {}", &email, nu.workspace_id),
+            format!(
+                "user {} declined invite to workspace {}",
+                &email, nu.workspace_id
+            ),
         ))
     } else {
         Err(Error::NotFound(format!("invite for {email} not found")))
@@ -727,7 +791,7 @@ async fn accept_invite(
     Json(nu): Json<AcceptInvite>,
 ) -> Result<(StatusCode, String)> {
     if &nu.username == "bot" {
-        return Err(Error::BadRequest("bot is a reserved username".to_string()))
+        return Err(Error::BadRequest("bot is a reserved username".to_string()));
     }
     let mut tx = db.begin().await?;
 
@@ -745,7 +809,7 @@ async fn accept_invite(
     }
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &nu.username,
         "users.accept_invite",
         ActionKind::Create,
@@ -759,14 +823,23 @@ async fn accept_invite(
     if is_admin.is_some() {
         Ok((
             StatusCode::CREATED,
-            format!("user {} accepted invite to workspace {}", &email, nu.workspace_id),
+            format!(
+                "user {} accepted invite to workspace {}",
+                &email, nu.workspace_id
+            ),
         ))
     } else {
         Err(Error::NotFound(format!("invite for {email} not found")))
     }
 }
 
-async fn add_user_to_workspace<'c>(w_id: &str, email: &str, username: &str, is_admin: bool, mut tx: sqlx::Transaction<'c, sqlx::Postgres>) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
+async fn add_user_to_workspace<'c>(
+    w_id: &str,
+    email: &str,
+    username: &str,
+    is_admin: bool,
+    mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
+) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
     sqlx::query!(
         "INSERT INTO usr
             (workspace_id, email, username, is_admin)
@@ -788,7 +861,7 @@ async fn add_user_to_workspace<'c>(w_id: &str, email: &str, username: &str, is_a
     .execute(&mut tx)
     .await?;
     audit_log(
-         &mut tx,
+        &mut tx,
         username,
         "users.add_to_workspace",
         ActionKind::Create,
@@ -811,7 +884,7 @@ async fn update_workspace_user(
     require_admin(is_admin, &username)?;
 
     if let Some(a) = eu.is_admin {
-            sqlx::query_scalar!(
+        sqlx::query_scalar!(
             "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3",
             a,
             &username_to_update,
@@ -837,6 +910,7 @@ async fn update_workspace_user(
 
 async fn update_user(
     Authed { email, .. }: Authed,
+    Path(email_to_update): Path<String>,
     Extension(db): Extension<DB>,
     Json(eu): Json<EditUser>,
 ) -> Result<String> {
@@ -845,10 +919,10 @@ async fn update_user(
     require_super_admin(&mut tx, email.clone()).await?;
 
     if let Some(sa) = eu.is_super_admin {
-            sqlx::query_scalar!(
+        sqlx::query_scalar!(
             "UPDATE password SET super_admin = $1 WHERE email = $2",
             sa,
-            &eu.email
+            &email_to_update
         )
         .execute(&mut tx)
         .await?;
@@ -860,12 +934,50 @@ async fn update_user(
         "users.update",
         ActionKind::Update,
         "global",
-        Some(&eu.email),
+        Some(&email_to_update),
         None,
     )
     .await?;
     tx.commit().await?;
-    Ok(format!("email {} updated", eu.email))
+    Ok(format!("email {} updated", &email_to_update))
+}
+
+async fn create_user(
+    Authed { email, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Extension(argon2): Extension<Arc<Argon2<'_>>>,
+    Json(nu): Json<NewUser>,
+) -> Result<(StatusCode, String)> {
+    let mut tx = db.begin().await?;
+
+    require_super_admin(&mut tx, email.clone()).await?;
+
+    sqlx::query!(
+        "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \
+         company)
+    VALUES ($1, $2, $3, 'password', $4, $5, $6)",
+        &nu.email,
+        true,
+        &hash_password(argon2, nu.password)?,
+        &nu.super_admin,
+        nu.name,
+        nu.company
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &email.unwrap(),
+        "users.update",
+        ActionKind::Update,
+        "global",
+        Some(&nu.email),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
 }
 
 pub fn owner_to_token_owner(user: &str, is_group: bool) -> String {
@@ -882,7 +994,6 @@ async fn delete_user(
 
     require_admin(is_admin, &username)?;
 
-
     let email_to_delete_o = sqlx::query_scalar!(
         "SELECT email FROM usr where username = $1 AND workspace_id = $2",
         username_to_delete,
@@ -909,7 +1020,6 @@ async fn delete_user(
     )
     .await?;
     tx.commit().await?;
-
     Ok(format!("username {} deleted", username_to_delete))
 }
 
@@ -920,18 +1030,24 @@ async fn set_password(
     Json(EditPassword { password }): Json<EditPassword>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email.ok_or("no_email").map_err(|e| Error::NotAuthorized(e.to_string()))?;
+    let email = email
+        .ok_or("no_email")
+        .map_err(|e| Error::NotAuthorized(e.to_string()))?;
 
     let custom_type = sqlx::query_scalar!(
-            "SELECT login_type::TEXT FROM password WHERE email = $1",
-        &email)
-        .fetch_one(&mut tx)
-        .await?
-        .unwrap_or("".to_string());
-    
+        "SELECT login_type::TEXT FROM password WHERE email = $1",
+        &email
+    )
+    .fetch_one(&mut tx)
+    .await
+    .map_err(|e| Error::InternalErr(format!("setting password: {e}")))?
+    .unwrap_or("".to_string());
+
     if custom_type != "password".to_string() {
-        return Err(Error::BadRequest(format!("login type for {email} is of type {custom_type}. Cannot set password.")))
-    } 
+        return Err(Error::BadRequest(format!(
+            "login type for {email} is of type {custom_type}. Cannot set password."
+        )));
+    }
 
     sqlx::query!(
         "UPDATE password SET password_hash = $1 WHERE email = $2",
@@ -972,7 +1088,6 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
     Ok(password_hash)
 }
 
-
 // async fn lost_password(
 //     Extension(db): Extension<DB>,
 //     Extension(es): Extension<Arc<EmailSender>>,
@@ -992,7 +1107,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 
 //     if !exists {
 //         return Err(Error::NotFound(format!("no user found at email {email}")))
-//     } 
+//     }
 
 //     let already = sqlx::query_scalar!(
 //             "SELECT EXISTS(SELECT 1 FROM magic_link WHERE email = $1)",
@@ -1042,7 +1157,6 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     }
 // }
 
-
 // async fn signup(
 //     TypedHeader(host): TypedHeader<headers::Host>,
 //     Extension(db): Extension<DB>,
@@ -1057,14 +1171,12 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 // ) -> Result<(StatusCode, String)> {
 //     let mut tx = db.begin().await?;
 
-
 //     let email = sqlx::query_scalar!(
 //             "INSERT INTO password (email, password_hash, name, company) VALUES ($1, $2, $3, $4) RETURNING email",
 //         &email, &hash_password(argon2, password)?, name, company)
 //         .fetch_optional(&mut tx)
 //         .await?;
 
-
 //     if let Some(email) = email {
 //         let tx = create_magic_link(&host.hostname(), &email, &es, tx).await?;
 //         tx.commit().await?;
@@ -1078,7 +1190,6 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     }
 // }
 
-
 // async fn create_magic_link<'c>(host: &str, email: &str, es: &EmailSender, mut tx: sqlx::Transaction<'c, sqlx::Postgres>) -> error::Result<sqlx::Transaction<'c, sqlx::Postgres>> {
 //     let token = gen_token();
 
@@ -1091,7 +1202,7 @@ pub fn hash_password(argon2: Arc<Argon2>, password: String) -> Result<String> {
 //     )
 //     .execute(&mut tx)
 //     .await?;
-    
+
 //     let encoded_token = urlencoding::encode(&token);
 //     let encoded_email = urlencoding::encode(email);
 //     es.send_email(Message::builder()
@@ -1117,20 +1228,17 @@ async fn login(
     cookies: Cookies,
     Extension(db): Extension<DB>,
     Extension(argon2): Extension<Arc<Argon2<'_>>>,
-    Json(Login {
-        email,
-        password,
-    }): Json<Login>,
+    Json(Login { email, password }): Json<Login>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
-
-    let email_w_h: Option<(String,  String, bool)> = sqlx::query_as(
-            "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = 'password'",
-        )
-        .bind(&email)
-        .fetch_optional(&mut tx)
-        .await?;
+    let email_w_h: Option<(String, String, bool)> = sqlx::query_as(
+        "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \
+         'password'",
+    )
+    .bind(&email)
+    .fetch_optional(&mut tx)
+    .await?;
 
     if let Some((email, hash, super_admin)) = email_w_h {
         let parsed_hash =
@@ -1150,7 +1258,12 @@ async fn login(
     }
 }
 
-pub async fn create_session_token<'c>(email: &str, super_admin: bool, tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, cookies: Cookies) -> Result<String> {
+pub async fn create_session_token<'c>(
+    email: &str,
+    super_admin: bool,
+    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    cookies: Cookies,
+) -> Result<String> {
     let token = gen_token();
     sqlx::query!(
         "INSERT INTO token
@@ -1198,8 +1311,11 @@ pub async fn create_token_for_owner(
         .map(char::from)
         .collect();
     let mut tx = db.begin().await?;
-    let is_super_admin = username.contains('@') && sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1",
-     owner.split_once('/').map(|x| x.1).unwrap_or(""))
+    let is_super_admin = username.contains('@')
+        && sqlx::query_scalar!(
+            "SELECT super_admin FROM password WHERE email = $1",
+            owner.split_once('/').map(|x| x.1).unwrap_or("")
+        )
         .fetch_optional(&mut tx)
         .await?
         .unwrap_or(false);
@@ -1244,16 +1360,19 @@ pub async fn create_token_for_owner(
 
 async fn create_token(
     Extension(db): Extension<DB>,
-    Authed { email,.. }: Authed,
+    Authed { email, .. }: Authed,
     Json(new_token): Json<NewToken>,
 ) -> Result<(StatusCode, String)> {
     let token = gen_token();
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| error::Error::BadRequest(format!("Only users with email can create tokens")))?;
-    let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
-        .fetch_optional(&mut tx)
-        .await?
-        .unwrap_or(false);
+    let email = email.ok_or_else(|| {
+        error::Error::BadRequest(format!("Only users with email can create tokens"))
+    })?;
+    let is_super_admin =
+        sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
+            .fetch_optional(&mut tx)
+            .await?
+            .unwrap_or(false);
     sqlx::query!(
         "INSERT INTO token
             (token, email, label, expiration, super_admin)
@@ -1287,8 +1406,8 @@ async fn list_tokens(
 ) -> JsonResult<Vec<TruncatedToken>> {
     let rows = sqlx::query_as!(
         TruncatedToken,
-        "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at FROM token \
-         WHERE email = $1",
+        "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \
+         last_used_at FROM token WHERE email = $1",
         email,
     )
     .fetch_all(&db)
@@ -1302,7 +1421,9 @@ async fn delete_token(
     Path(token_prefix): Path<String>,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
-    let email = email.ok_or_else(|| error::Error::BadRequest(format!("Only users with email can create tokens")))?;
+    let email = email.ok_or_else(|| {
+        error::Error::BadRequest(format!("Only users with email can create tokens"))
+    })?;
     let tokens_deleted: Vec<String> = sqlx::query_scalar(
         "DELETE FROM token WHERE email = $1 AND
      token LIKE concat($3, '%') RETURNING concat(substring(token for 10), '*****')",
@@ -1347,13 +1468,13 @@ async fn leave_workspace(
     .await?;
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &username,
         "users.leave_workspace",
         ActionKind::Delete,
         &w_id,
-None,
-None,
+        None,
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -1361,7 +1482,6 @@ None,
     Ok(format!("left workspace {w_id}"))
 }
 
-
 pub async fn delete_expired_items_perdiodically(
     db: &DB,
     mut rx: tokio::sync::broadcast::Receiver<()>,
@@ -1375,12 +1495,11 @@ pub async fn delete_expired_items_perdiodically(
         .fetch_all(db)
         .await;
 
-
         match tokens_deleted_r {
-            Ok(tokens) => tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens),
+            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
             Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
-        
+
         let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
             "DELETE FROM magic_link WHERE expiration <= $1
         RETURNING concat(substring(token for 10), '*****')",
@@ -1390,7 +1509,7 @@ pub async fn delete_expired_items_perdiodically(
         .await;
 
         match magic_links_deleted_r {
-            Ok(tokens) => tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens),
+            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
             Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
         }
 

backend/src/utils.rs

@@ -1,18 +1,20 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use reqwest::Response;
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
 
-use crate::error::{Error, Result};
+use crate::error::{to_anyhow, Error, Result};
 
 pub const MAX_PER_PAGE: usize = 1000;
-pub const DEFAULT_PER_PAGE: usize = 30;
+pub const DEFAULT_PER_PAGE: usize = 100;
 
 #[derive(Deserialize)]
 pub struct Pagination {
@@ -20,11 +22,15 @@ pub struct Pagination {
     pub per_page: Option<usize>,
 }
 #[derive(Deserialize)]
-pub struct StripPath(String);
+pub struct StripPath(pub String);
 
 impl StripPath {
     pub fn to_path(&self) -> &str {
-        self.0.strip_prefix('/').unwrap()
+        if self.0.starts_with('/') {
+            self.0.strip_prefix('/').unwrap()
+        } else {
+            &self.0
+        }
     }
 }
 
@@ -37,7 +43,8 @@ pub async fn require_super_admin<'c>(
         email.as_ref()
     )
     .fetch_one(db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
     if !is_admin {
         Err(Error::NotAuthorized(
             "This endpoint require caller to be a super admin".to_owned(),
@@ -91,3 +98,46 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
 pub fn get_owner_from_path(path: &str) -> String {
     path.split('/').take(2).collect::<Vec<_>>().join("/")
 }
+
+pub async fn list_elems_from_hub(
+    http_client: reqwest::Client,
+    url: &str,
+    email: Option<String>,
+    username: String,
+    host: String,
+) -> Result<serde_json::Value> {
+    let rows = http_get_from_hub(http_client, url, email, username, host, false)
+        .await?
+        .json::<serde_json::Value>()
+        .await
+        .map_err(to_anyhow)?;
+    Ok(rows)
+}
+
+pub async fn http_get_from_hub(
+    http_client: reqwest::Client,
+    url: &str,
+    email: Option<String>,
+    username: String,
+    host: String,
+    plain: bool,
+) -> Result<Response> {
+    let response = http_client
+        .get(url)
+        .header(
+            "Accept",
+            if plain {
+                "text/plain"
+            } else {
+                "application/json"
+            },
+        )
+        .header("X-email", email.unwrap_or_else(|| "".to_string()))
+        .header("X-username", username)
+        .header("X-hostname", host)
+        .send()
+        .await
+        .map_err(to_anyhow)?;
+
+    Ok(response)
+}

backend/src/variables.rs

@@ -1,14 +1,18 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::sync::Arc;
+
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{Error, JsonResult, Result},
+    oauth2::{AllClients, _refresh_token},
     users::Authed,
     utils::StripPath,
 };
@@ -20,6 +24,7 @@ use axum::{
 use hyper::StatusCode;
 
 use magic_crypt::{MagicCrypt256, MagicCryptTrait};
+use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use sqlx::{FromRow, Postgres, Transaction};
 
@@ -28,6 +33,7 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_variables))
         .route("/list_contextual", get(list_contextual_variables))
         .route("/get/*path", get(get_variable))
+        .route("/exists/*path", get(exists_variable))
         .route("/update/*path", post(update_variable))
         .route("/delete/*path", delete(delete_variable))
         .route("/create", post(create_variable))
@@ -50,6 +56,9 @@ pub struct ListableVariable {
     pub is_secret: bool,
     pub description: String,
     pub extra_perms: serde_json::Value,
+    pub account: Option<i32>,
+    pub is_oauth: bool,
+    pub is_expired: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -58,6 +67,8 @@ pub struct CreateVariable {
     pub value: String,
     pub is_secret: bool,
     pub description: String,
+    pub account: Option<i32>,
+    pub is_oauth: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -74,41 +85,67 @@ pub fn get_reserved_variables(
     email: &str,
     username: &str,
     job_id: &str,
-) -> [ContextualVariable; 5] {
+    permissioned_as: &str,
+    path: Option<String>,
+    flow_path: Option<String>,
+    schedule_path: Option<String>,
+) -> [ContextualVariable; 9] {
     [
         ContextualVariable {
             name: "WM_WORKSPACE".to_string(),
             value: w_id.to_string(),
-            description: "Workspace id of the current script".to_string()
+            description: "Workspace id of the current script".to_string(),
         },
         ContextualVariable {
             name: "WM_TOKEN".to_string(),
             value: token.to_string(),
-            description: "Token ephemeral to the current script with equal permission to the permission of the run (Usable as a bearer token)".to_string()
+            description: "Token ephemeral to the current script with equal permission to the \
+                          permission of the run (Usable as a bearer token)"
+                .to_string(),
         },
         ContextualVariable {
             name: "WM_EMAIL".to_string(),
             value: email.to_string(),
-            description: "Email of the user that executed the current script".to_string()
+            description: "Email of the user that executed the current script".to_string(),
         },
         ContextualVariable {
             name: "WM_USERNAME".to_string(),
             value: username.to_string(),
-            description: "Username of the user that executed the current script".to_string()
+            description: "Username of the user that executed the current script".to_string(),
         },
         ContextualVariable {
             name: "WM_JOB_ID".to_string(),
             value: job_id.to_string(),
-            description: "Job id of the current script".to_string()
+            description: "Job id of the current script".to_string(),
+        },
+        ContextualVariable {
+            name: "WM_JOB_PATH".to_string(),
+            value: path.unwrap_or_else(|| "".to_string()),
+            description: "Path of the script or flow being run if any".to_string(),
+        },
+        ContextualVariable {
+            name: "WM_FLOW_PATH".to_string(),
+            value: flow_path.unwrap_or_else(|| "".to_string()),
+            description: "Path of the encapsulating flow if the job is a flow step".to_string(),
+        },
+        ContextualVariable {
+            name: "WM_SCHEDULE_PATH".to_string(),
+            value: schedule_path.unwrap_or_else(|| "".to_string()),
+            description: "Path of the schedule if the job of the step or encapsulating step has \
+                          been triggered by a schedule"
+                .to_string(),
+        },
+        ContextualVariable {
+            name: "WM_PERMISSIONED_AS".to_string(),
+            value: permissioned_as.to_string(),
+            description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
         },
     ]
 }
 
 async fn list_contextual_variables(
     Path(w_id): Path<String>,
-    Authed {
-        username, email, ..
-    }: Authed,
+    Authed { username, email, .. }: Authed,
 ) -> JsonResult<Vec<ContextualVariable>> {
     Ok(Json(
         get_reserved_variables(
@@ -117,6 +154,10 @@ async fn list_contextual_variables(
             &email.unwrap_or_else(|| "no email".to_string()),
             &username,
             "017e0ad5-f499-73b6-5488-92a61c5196dd",
+            format!("u/{username}").as_str(),
+            Some("u/user/script_path".to_string()),
+            Some("u/user/encapsulating_flow_path".to_string()),
+            Some("u/user/triggering_flow_path".to_string()),
         )
         .to_vec(),
     ))
@@ -130,8 +171,11 @@ async fn list_variables(
     let mut tx = user_db.begin(&authed).await?;
 
     let rows = sqlx::query_as::<_, ListableVariable>(
-        "SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as value, is_secret, description, extra_perms from variable
-         WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER BY path",
+        "SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as \
+         value, is_secret, description, extra_perms, account, is_oauth, false as is_expired from \
+         variable
+         WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER \
+         BY path",
     )
     .bind(&w_id)
     .fetch_all(&mut tx)
@@ -151,12 +195,18 @@ async fn get_variable(
     Extension(user_db): Extension<UserDB>,
     Query(q): Query<GetVariableQuery>,
     Path((w_id, path)): Path<(String, StripPath)>,
+    Extension(clients): Extension<Arc<AllClients>>,
+    Extension(http_client): Extension<Client>,
 ) -> JsonResult<ListableVariable> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
     let variable_o = sqlx::query_as::<_, ListableVariable>(
-        "SELECT * from variable WHERE path = $1 AND (workspace_id = $2 OR (is_secret IS NOT TRUE AND workspace_id = 'starter'))",
+        "SELECT variable.*, (now() > account.expires_at) as is_expired from variable
+        LEFT JOIN account ON variable.account = account.id
+        WHERE variable.path = $1 AND (variable.workspace_id = $2 OR (is_secret IS NOT TRUE AND \
+         variable.workspace_id = 'starter')) 
+        LIMIT 1",
     )
     .bind(&path)
     .bind(&w_id)
@@ -180,8 +230,22 @@ async fn get_variable(
         .await?;
         let value = variable.value.unwrap_or_else(|| "".to_string());
         ListableVariable {
-            value: if !value.is_empty() && decrypt_secret {
+            value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
+                Some(
+                    _refresh_token(
+                        tx,
+                        &variable.path,
+                        w_id,
+                        variable.account.unwrap(),
+                        clients,
+                        http_client,
+                    )
+                    .await?,
+                )
+            } else if !value.is_empty() && decrypt_secret {
                 let mc = build_crypt(&mut tx, &w_id).await?;
+                tx.commit().await?;
+
                 Some(
                     mc.decrypt_base64_to_string(value)
                         .map_err(|e| Error::InternalErr(e.to_string()))?,
@@ -194,11 +258,28 @@ async fn get_variable(
     } else {
         variable
     };
-    tx.commit().await?;
 
     Ok(Json(r))
 }
 
+async fn exists_variable(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
 async fn create_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -209,20 +290,22 @@ async fn create_variable(
 
     let value = if variable.is_secret {
         let mc = build_crypt(&mut tx, &w_id).await?;
-        encrypt(&mc, variable.value)
+        encrypt(&mc, &variable.value)
     } else {
         variable.value
     };
 
     sqlx::query!(
         "INSERT INTO variable
-            (workspace_id, path, value, is_secret, description)
-            VALUES ($1, $2, $3, $4, $5)",
+            (workspace_id, path, value, is_secret, description, account, is_oauth)
+            VALUES ($1, $2, $3, $4, $5, $6, $7)",
         &w_id,
         variable.path,
         value,
         variable.is_secret,
-        variable.description
+        variable.description,
+        variable.account,
+        variable.is_oauth.unwrap_or(false),
     )
     .execute(&mut tx)
     .await?;
@@ -249,7 +332,6 @@ async fn create_variable(
 async fn delete_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -260,7 +342,7 @@ async fn delete_variable(
         path,
         w_id
     )
-    .execute(&db)
+    .execute(&mut tx)
     .await?;
     audit_log(
         &mut tx,
@@ -281,7 +363,6 @@ async fn delete_variable(
 async fn update_variable(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditVariable>,
 ) -> Result<String> {
@@ -310,7 +391,7 @@ async fn update_variable(
 
         let value = if is_secret {
             let mc = build_crypt(&mut tx, &w_id).await?;
-            encrypt(&mc, nvalue)
+            encrypt(&mc, &nvalue)
         } else {
             nvalue
         };
@@ -329,9 +410,14 @@ async fn update_variable(
         }
         sqlb.set_str("is_secret", nbool);
     }
+    sqlb.returning("path");
+
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
 
-    sqlx::query(&sql).execute(&db).await?;
+    let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+
+    let npath = crate::utils::not_found_if_none(npath_o, "Variable", path)?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -344,7 +430,7 @@ async fn update_variable(
     .await?;
     tx.commit().await?;
 
-    Ok(format!("variable {} updated (npath: {:?})", path, ns.path))
+    Ok(format!("variable {} updated (npath: {:?})", path, npath))
 }
 
 pub async fn build_crypt<'c>(
@@ -356,10 +442,12 @@ pub async fn build_crypt<'c>(
         w_id
     )
     .fetch_one(db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching crypt key: {e}")))?;
+
     Ok(magic_crypt::new_magic_crypt!(key, 256))
 }
 
-pub fn encrypt(mc: &MagicCrypt256, value: String) -> String {
+pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String {
     mc.encrypt_str_to_base64(value)
 }

backend/src/worker.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
@@ -7,6 +8,7 @@
 
 use itertools::Itertools;
 use std::{
+    collections::HashMap,
     process::{ExitStatus, Stdio},
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -19,14 +21,16 @@ use crate::{
     db::DB,
     error::Error,
     jobs::{
-        add_completed_job, add_completed_job_error, handle_flow, postprocess_queued_job, pull,
-        update_flow_status_after_job_completion, update_flow_status_in_progress, JobKind,
+        add_completed_job, add_completed_job_error, postprocess_queued_job, pull, JobKind,
         QueuedJob,
     },
     parser::{self, Typ},
-    scripts::ScriptHash,
+    scripts::{ScriptHash, ScriptLang},
     users::{create_token_for_owner, get_email_from_username},
     variables,
+    worker_flow::{
+        handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress,
+    },
 };
 
 use serde_json::{json, Map, Value};
@@ -44,11 +48,14 @@ use tokio::sync::mpsc;
 
 const TMP_DIR: &str = "/tmp/windmill";
 const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip";
+const DENO_CACHE_DIR: &str = "/tmp/windmill/cache/deno";
 const NUM_SECS_ENV_CHECK: u64 = 15;
 
 const INCLUDE_DEPS_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.sh");
 const NSJAIL_CONFIG_DOWNLOAD_CONTENT: &str = include_str!("../../nsjail/download.config.proto");
-const NSJAIL_CONFIG_RUN_CONTENT: &str = include_str!("../../nsjail/run.config.proto");
+const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str =
+    include_str!("../../nsjail/run.python3.config.proto");
+const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../../nsjail/run.deno.config.proto");
 
 pub async fn run_worker(
     db: &DB,
@@ -61,22 +68,20 @@ pub async fn run_worker(
     ip: &str,
     sleep_queue: u64,
     base_url: &str,
+    disable_nuser: bool,
+    disable_nsjail: bool,
     tx: tokio::sync::broadcast::Sender<()>,
 ) {
     let worker_dir = format!("{TMP_DIR}/{worker_name}");
     tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
 
-    DirBuilder::new()
-        .recursive(true)
-        .create(&worker_dir)
-        .await
-        .expect("could not create initial worker dir");
-
-    DirBuilder::new()
-        .recursive(true)
-        .create(&PIP_CACHE_DIR)
-        .await
-        .expect("could not create initial worker dir");
+    for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR] {
+        DirBuilder::new()
+            .recursive(true)
+            .create(x)
+            .await
+            .expect("could not create initial worker dir");
+    }
 
     let _ = write_file(&worker_dir, "download_deps.sh", INCLUDE_DEPS_SH_CONTENT).await;
 
@@ -107,23 +112,47 @@ pub async fn run_worker(
 
                 tracing::info!(worker = %worker_name, id = %job.id, "Fetched job");
                 let job2 = job.clone();
-                if let Some(err) =
-                    handle_queued_job(job, db, timeout, &worker_name, &worker_dir, base_url)
-                        .await
-                        .err()
+                if let Some(err) = handle_queued_job(
+                    job,
+                    db,
+                    timeout,
+                    &worker_name,
+                    &worker_dir,
+                    base_url,
+                    disable_nuser,
+                    disable_nsjail,
+                )
+                .await
+                .err()
                 {
                     let err_string = err.to_string().clone();
-                    let _ = add_completed_job_error(
+                    let m = add_completed_job_error(
                         db,
                         &job2,
                         "Unexpected error during job execution:\n".to_string(),
                         err,
                     )
-                    .await;
+                    .await
+                    .map(|(_, m)| m)
+                    .unwrap_or_else(|_| Map::new());
 
-                    let _ =
-                        postprocess_queued_job(job2.schedule_path, &job2.workspace_id, job2.id, db)
+                    {
+                        let job = job2.clone();
+                        let _ = postprocess_queued_job(
+                            job.is_flow_step,
+                            job.schedule_path,
+                            job.script_path,
+                            &job2.workspace_id,
+                            job2.id,
+                            db,
+                        )
+                        .await;
+                    }
+
+                    if job2.parent_job.is_some() {
+                        let _ = update_flow_status_after_job_completion(db, &job2, false, Some(m))
                             .await;
+                    }
                     tracing::error!(job_id = %job2.id, "Error handling job: {err_string}");
                 };
             }
@@ -162,6 +191,8 @@ async fn handle_queued_job(
     worker_name: &str,
     worker_dir: &str,
     base_url: &str,
+    disable_nuser: bool,
+    disable_nsjail: bool,
 ) -> crate::error::Result<()> {
     let job_id = job.id;
     let w_id = &job.workspace_id.clone();
@@ -198,12 +229,14 @@ async fn handle_queued_job(
                 &mut logs,
                 &mut last_line,
                 base_url,
+                disable_nuser,
+                disable_nsjail,
             )
             .await;
 
             match execution {
                 Ok(r) => {
-                    add_completed_job(db, &job, true, r.result.clone(), logs).await?;
+                    add_completed_job(db, &job, true, false, r.result.clone(), logs).await?;
                     if job.is_flow_step {
                         update_flow_status_after_job_completion(db, &job, true, r.result).await?;
                     }
@@ -217,7 +250,15 @@ async fn handle_queued_job(
                 }
             };
 
-            let _ = postprocess_queued_job(job.schedule_path, &w_id, job_id, db).await;
+            let _ = postprocess_queued_job(
+                job.is_flow_step,
+                job.schedule_path,
+                job.script_path,
+                &w_id,
+                job_id,
+                db,
+            )
+            .await;
         }
     }
     Ok(())
@@ -246,6 +287,9 @@ async fn transform_json_value(token: &str, workspace: &str, base_url: &str, v: V
         }
         Value::String(y) if y.starts_with("$res:") => {
             let path = y.strip_prefix("$res:").unwrap();
+            if path.split("/").count() < 2 {
+                return Value::String(format!("resource path: {path} is ill-defined"));
+            }
             let v = crate::client::get_resource(workspace, path, token, base_url)
                 .await
                 .ok()
@@ -270,9 +314,11 @@ async fn handle_job(
     timeout: i32,
     worker_name: &str,
     worker_dir: &str,
-    mut logs: &mut String,
-    mut last_line: &mut String,
+    logs: &mut String,
+    last_line: &mut String,
     base_url: &str,
+    disable_nuser: bool,
+    disable_nsjail: bool,
 ) -> Result<JobResult, Error> {
     tracing::info!(
         worker = %worker_name,
@@ -288,111 +334,297 @@ async fn handle_job(
         .await
         .expect("could not create initial job dir");
 
-    let mut status: Result<ExitStatus, Error>;
+    let mut status: Result<ExitStatus, Error> =
+        Err(Error::InternalErr("job not started".to_string()));
+
     if matches!(job.job_kind, JobKind::Dependencies) {
-        let requirements = job
-            .raw_code
-            .as_ref()
-            .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?;
-        logs.push_str(&format!("content of requirements:\n{}\n", &requirements));
-
-        let file = "requirements.in";
-        write_file(&job_dir, file, &requirements).await?;
-
-        let child = Command::new("pip-compile")
-            .current_dir(&job_dir)
-            .args(vec!["-q", "--no-header", file])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
-
-        status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
-
-        if status.is_ok() && status.as_ref().unwrap().success() {
-            let path_lock = format!("{}/requirements.txt", job_dir);
-            let mut file = File::open(path_lock).await?;
-
-            let mut content = "".to_string();
-            file.read_to_string(&mut content).await?;
-            content = content
-                .lines()
-                .filter(|x| !x.trim_start().starts_with('#'))
-                .map(|x| x.to_string())
-                .collect::<Vec<String>>()
-                .join("\n");
-            let as_json = json!(content);
-
-            *last_line =
-                format!(r#"{{ "success": "Successful lock file generation", "lock": {as_json} }}"#);
-
-            sqlx::query!(
-                "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
-                &content,
-                &job.script_hash.unwrap_or(ScriptHash(0)).0,
-                &job.workspace_id
-            )
-            .execute(db)
-            .await?;
-        } else {
-            sqlx::query!(
-                "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3",
-                &logs.clone(),
-                &job.script_hash.unwrap_or(ScriptHash(0)).0,
-                &job.workspace_id
-            )
-            .execute(db)
-            .await?;
-        }
+        handle_dependency_job(job, logs, &job_dir, &mut status, db, last_line, timeout).await?;
     } else {
-        let (inner_content, requirements_o) = if matches!(job.job_kind, JobKind::Preview) {
-            let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
-            let reqs = parser::parse_imports(&code)?.join("\n");
-            (code, Some(reqs))
-        } else {
-            sqlx::query_as::<_, (String, Option<String>)>("SELECT content, lock FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')")
-            .bind(&job.script_hash.unwrap_or(ScriptHash(0)).0)
-            .bind(&job.workspace_id)
-            .fetch_optional(db)
-            .await?
-            .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?
-        };
-
-        let requirements =
-            requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?;
-
-        let _ = write_file(
+        handle_nondep_job(
+            job,
+            db,
             &job_dir,
-            "download.config.proto",
-            &NSJAIL_CONFIG_DOWNLOAD_CONTENT
-                .replace("{JOB_DIR}", &job_dir)
-                .replace("{WORKER_DIR}", &worker_dir)
-                .replace("{CACHE_DIR}", PIP_CACHE_DIR),
+            worker_dir,
+            disable_nuser,
+            disable_nsjail,
+            logs,
+            &mut status,
+            last_line,
+            timeout,
+            base_url,
         )
         .await?;
-        let _ = write_file(&job_dir, "requirements.txt", &requirements).await?;
+    }
+    tokio::fs::remove_dir_all(job_dir).await?;
 
-        let child = Command::new("nsjail")
-            .current_dir(&job_dir)
-            .args(vec!["--config", "download.config.proto"])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
+    if status.is_ok() && status.as_ref().unwrap().success() {
+        let result = serde_json::from_str::<Map<String, Value>>(last_line).map_err(|e| {
+            Error::ExecutionErr(format!(
+                "result {} is not parsable.\n err: {}",
+                last_line,
+                e.to_string()
+            ))
+        })?;
+        Ok(JobResult { result: Some(result) })
+    } else {
+        let err = match status {
+            Ok(_) => {
+                let s = format!(
+                    "Error during execution of the script\nlast 5 logs lines:\n{}",
+                    logs.lines()
+                        .skip(logs.lines().count().max(5) - 5)
+                        .join("\n")
+                );
+                logs.push_str("\n\n--- ERROR ---\n");
+                s
+            }
+            Err(err) => format!("error before termination: {err}"),
+        };
+        Err(Error::ExecutionErr(err))
+    }
+}
 
-        logs.push_str("\n--- DEPENDENCIES INSTALL ---\n");
-        status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
-        if status.is_ok() {
-            logs.push_str("\n\n--- CODE EXECUTION ---\n");
+async fn handle_nondep_job(
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    job_dir: &String,
+    worker_dir: &str,
+    disable_nuser: bool,
+    disable_nsjail: bool,
+    logs: &mut String,
+    status: &mut Result<ExitStatus, Error>,
+    last_line: &mut String,
+    timeout: i32,
+    base_url: &str,
+) -> Result<(), Error> {
+    let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview)
+        || matches!(job.job_kind, JobKind::Script_Hub)
+    {
+        let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
+        let reqs = if job
+            .language
+            .as_ref()
+            .map(|x| matches!(x, ScriptLang::Python3))
+            .unwrap_or(false)
+        {
+            Some(parser::parse_python_imports(&code)?.join("\n"))
+        } else {
+            None
+        };
+        (code, reqs, job.language.to_owned())
+    } else {
+        sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>)>(
+            "SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR \
+             workspace_id = 'starter')",
+        )
+        .bind(&job.script_hash.unwrap_or(ScriptHash(0)).0)
+        .bind(&job.workspace_id)
+        .fetch_optional(db)
+        .await?
+        .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?
+    };
+    match language {
+        None => {
+            return Err(Error::ExecutionErr(
+                "Require language to be not null".to_string(),
+            ))?;
+        }
+        Some(ScriptLang::Python3) => {
+            let requirements =
+                requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?;
+
+            if !disable_nsjail {
+                let _ = write_file(
+                    job_dir,
+                    "download.config.proto",
+                    &NSJAIL_CONFIG_DOWNLOAD_CONTENT
+                        .replace("{JOB_DIR}", job_dir)
+                        .replace("{WORKER_DIR}", &worker_dir)
+                        .replace("{CACHE_DIR}", PIP_CACHE_DIR)
+                        .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()),
+                )
+                .await?;
+            }
+            let _ = write_file(job_dir, "requirements.txt", &requirements).await?;
+
+            let child = if !disable_nsjail {
+                Command::new("nsjail")
+                    .current_dir(job_dir)
+                    .args(vec!["--config", "download.config.proto"])
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .spawn()?
+            } else {
+                Command::new("/usr/local/bin/python3")
+                    .current_dir(job_dir)
+                    .args(vec![
+                        "-m",
+                        "pip",
+                        "install",
+                        "--no-color",
+                        "--isolated",
+                        "--no-warn-conflicts",
+                        "--disable-pip-version-check",
+                        "-t",
+                        "./dependencies",
+                        "-r",
+                        "./requirements.txt",
+                    ])
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .spawn()?
+            };
+
+            logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n");
+            *status = handle_child(job, db, logs, last_line, timeout, child).await;
+
+            if status.is_ok() {
+                logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
+
+                set_logs(logs, job.id, db).await;
+
+                let _ = write_file(job_dir, "inner.py", &inner_content).await?;
+
+                let sig = crate::parser::parse_python_signature(&inner_content)?;
+                let transforms = sig
+                    .args
+                    .into_iter()
+                    .map(|x| match x.typ {
+                        Typ::Bytes => {
+                            format!(
+                                "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    \
+                                     kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n",
+                                x.name, x.name, x.name, x.name
+                            )
+                        }
+                        Typ::Datetime => {
+                            format!(
+                                "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    \
+                                     kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], \
+                                     '%Y-%m-%dT%H:%M')\n",
+                                x.name, x.name, x.name, x.name
+                            )
+                        }
+                        _ => "".to_string(),
+                    })
+                    .collect::<Vec<String>>()
+                    .join("");
+
+                let tx = db.begin().await?;
+
+                let token = create_token_for_owner(
+                    &db,
+                    &job.workspace_id,
+                    &job.permissioned_as,
+                    crate::users::NewToken {
+                        label: Some("ephemeral-script".to_string()),
+                        expiration: Some(
+                            chrono::Utc::now() + chrono::Duration::seconds((timeout * 2).into()),
+                        ),
+                    },
+                    &job.created_by,
+                )
+                .await?;
+
+                let args = if let Some(args) = &job.args {
+                    Some(
+                        transform_json_value(&token, &job.workspace_id, base_url, args.clone())
+                            .await,
+                    )
+                } else {
+                    None
+                };
+                let ser_args =
+                    serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?;
+                write_file(job_dir, "args.json", &ser_args).await?;
+
+                let wrapper_content: String = format!(
+                    r#"
+import json
+import base64
+from datetime import datetime
+
+inner_script = __import__("inner")
+
+with open("args.json") as f:
+    kwargs = json.load(f, strict=False)
+for k, v in kwargs.items():
+    if v == '<function call>':
+        kwargs[k] = None
+{transforms}
+res = inner_script.main(**kwargs)
+if res is None:
+    res = {{}}
+if isinstance(res, tuple):
+    res = {{f"res{{i+1}}": v for i, v in enumerate(res)}}
+if not isinstance(res, dict):
+    res = {{ "res1": res }}
+res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
+print()
+print("result:")
+print(res_json)
+"#,
+                );
+                write_file(job_dir, "main.py", &wrapper_content).await?;
+
+                tx.commit().await?;
+                let mut reserved_variables = get_reserved_variables(job, token, db).await?;
+                if !disable_nuser {
+                    let _ = write_file(
+                        job_dir,
+                        "run.config.proto",
+                        &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
+                            .replace("{JOB_DIR}", job_dir)
+                            .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()),
+                    )
+                    .await?;
+                } else {
+                    reserved_variables
+                        .insert("PYTHONPATH".to_string(), format!("{job_dir}/dependencies"));
+                }
+
+                let child = if !disable_nuser {
+                    Command::new("nsjail")
+                        .current_dir(job_dir)
+                        .env_clear()
+                        .envs(reserved_variables)
+                        .args(vec![
+                            "--config",
+                            "run.config.proto",
+                            "--",
+                            "/usr/local/bin/python3",
+                            "-u",
+                            "/tmp/main.py",
+                        ])
+                        .stdout(Stdio::piped())
+                        .stderr(Stdio::piped())
+                        .spawn()?
+                } else {
+                    Command::new("/usr/local/bin/python3")
+                        .current_dir(job_dir)
+                        .env_clear()
+                        .envs(reserved_variables)
+                        .args(vec!["-u", "main.py"])
+                        .stdout(Stdio::piped())
+                        .stderr(Stdio::piped())
+                        .spawn()?
+                };
+                *status = handle_child(job, db, logs, last_line, timeout, child).await;
+            }
+        }
+        Some(ScriptLang::Deno) => {
+            logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
 
             set_logs(logs, job.id, db).await;
 
-            let _ = write_file(&job_dir, "inner.py", &inner_content).await?;
+            let _ = write_file(job_dir, "inner.ts", &inner_content).await?;
 
-            let sig = crate::parser::parse_signature(&inner_content)?;
-            let transforms = sig.args.into_iter().map(|x| match x.typ {
-        Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
-        Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
-        _ => "".to_string()
-    }).collect::<Vec<String>>().join("");
+            let sig = crate::parser::parse_deno_signature(&inner_content)?;
+            //             let transforms = sig.args.clone().into_iter().map(|x| match x.typ {
+            //     Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
+            //     Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
+            //     _ => "".to_string()
+            // }).collect::<Vec<String>>().join("");
 
             let tx = db.begin().await?;
 
@@ -415,103 +647,187 @@ async fn handle_job(
             } else {
                 None
             };
-            let ser_args = serde_json::to_string(&args)
-                .map_err(|e| Error::ExecutionErr(e.to_string()))?
-                .replace("\\\"", "\\\\\"");
+            let ser_args =
+                serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?;
+            write_file(job_dir, "args.json", &ser_args).await?;
+
+            let spread = sig.args.into_iter().map(|x| x.name).join(",");
             let wrapper_content: String = format!(
                 r#"
-import json
-import base64
-from datetime import datetime
+import {{ main }} from "./inner.ts";
 
-inner_script = __import__("inner")
+const args = await Deno.readTextFile("args.json")
+    .then(JSON.parse)
+    .then(({{ {spread} }}) => [ {spread} ])
 
-kwargs = json.loads("""{ser_args}""", strict=False)
-for k, v in kwargs.items():
-    if v == '<function call>':
-        kwargs[k] = None
-{transforms}
-res = inner_script.main(**kwargs)
-if res is None:
-    res = {{}}
-if isinstance(res, tuple):
-    res = {{f"res{{i+1}}": v for i, v in enumerate(res)}}
-if not isinstance(res, dict):
-    res = {{ "res1": res }}
-res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
-print()
-print("result:")
-print(res_json)
+async function run() {{
+    let res: any = await main(...args);
+    if (res == undefined) {{
+        res = {{}}
+    }}
+    if (typeof res !== 'object' || Array.isArray(res)) {{
+        res = {{ res1: res }}
+    }}
+
+    const res_json = JSON.stringify(res);
+    console.log();
+    console.log("result:");
+    console.log(res_json);
+    Deno.exit(0);
+}}
+run();
 "#,
             );
-            write_file(&job_dir, "main.py", &wrapper_content).await?;
+            write_file(job_dir, "main.ts", &wrapper_content).await?;
 
             tx.commit().await?;
-            let reserved_variables = variables::get_reserved_variables(
-                &job.workspace_id,
-                &token,
-                &get_email_from_username(&job.created_by, db)
-                    .await?
-                    .unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
-                &job.created_by,
-                &job.id.to_string(),
-            )
-            .into_iter()
-            .map(|rv| (rv.name, rv.value));
 
-            let _ = write_file(
-                &job_dir,
-                "run.config.proto",
-                &NSJAIL_CONFIG_RUN_CONTENT.replace("{JOB_DIR}", &job_dir),
-            )
-            .await?;
+            let mut reserved_variables = get_reserved_variables(job, token, db).await?;
+            reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
 
-            let child = Command::new("nsjail")
-                .current_dir(&job_dir)
-                .envs(reserved_variables)
-                .args(vec![
-                    "--config",
+            if !disable_nuser {
+                let _ = write_file(
+                    job_dir,
                     "run.config.proto",
-                    "--",
-                    "/usr/local/bin/python3",
-                    "-u",
-                    "/tmp/main.py",
-                ])
-                .stdout(Stdio::piped())
-                .stderr(Stdio::piped())
-                .spawn()?;
-            status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
+                    &NSJAIL_CONFIG_RUN_DENO_CONTENT
+                        .replace("{JOB_DIR}", job_dir)
+                        .replace("{CACHE_DIR}", DENO_CACHE_DIR)
+                        .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()),
+                )
+                .await?;
+            }
+
+            let child = if !disable_nsjail {
+                Command::new("nsjail")
+                    .current_dir(job_dir)
+                    .env_clear()
+                    .envs(reserved_variables)
+                    .args(vec![
+                        "--config",
+                        "run.config.proto",
+                        "--",
+                        "/usr/bin/deno",
+                        "run",
+                        "--unstable",
+                        "--v8-flags=--max-heap-size=2048",
+                        "-A",
+                        "/tmp/main.ts",
+                    ])
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .spawn()?
+            } else {
+                Command::new("/usr/bin/deno")
+                    .current_dir(job_dir)
+                    .env_clear()
+                    .envs(reserved_variables)
+                    .args(vec![
+                        "run",
+                        "--unstable",
+                        "--v8-flags=--max-heap-size=2048",
+                        "-A",
+                        "main.ts",
+                    ])
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .spawn()?
+            };
+            *status = handle_child(job, db, logs, last_line, timeout, child).await;
         }
     }
-    tokio::fs::remove_dir_all(job_dir).await?;
+    Ok(())
+}
 
-    if status.is_ok() && status.as_ref().unwrap().success() {
-        let result = serde_json::from_str::<Map<String, Value>>(last_line).map_err(|e| {
-            Error::ExecutionErr(format!(
-                "result {} is not parsable.\n err: {}",
-                last_line,
-                e.to_string()
-            ))
-        })?;
-        Ok(JobResult {
-            result: Some(result),
-        })
+async fn handle_dependency_job(
+    job: &QueuedJob,
+    logs: &mut String,
+    job_dir: &String,
+    status: &mut Result<ExitStatus, Error>,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    last_line: &mut String,
+    timeout: i32,
+) -> Result<(), Error> {
+    let requirements = job
+        .raw_code
+        .as_ref()
+        .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?;
+    logs.push_str(&format!("content of requirements:\n{}\n", &requirements));
+    let file = "requirements.in";
+    write_file(job_dir, file, &requirements).await?;
+    let child = Command::new("pip-compile")
+        .current_dir(job_dir)
+        .args(vec!["-q", "--no-header", file])
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+    *status = handle_child(job, db, logs, last_line, timeout, child).await;
+    Ok(if status.is_ok() && status.as_ref().unwrap().success() {
+        let path_lock = format!("{}/requirements.txt", job_dir);
+        let mut file = File::open(path_lock).await?;
+
+        let mut content = "".to_string();
+        file.read_to_string(&mut content).await?;
+        content = content
+            .lines()
+            .filter(|x| !x.trim_start().starts_with('#'))
+            .map(|x| x.to_string())
+            .collect::<Vec<String>>()
+            .join("\n");
+        let as_json = json!(content);
+
+        *last_line =
+            format!(r#"{{ "success": "Successful lock file generation", "lock": {as_json} }}"#);
+
+        sqlx::query!(
+            "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
+            &content,
+            &job.script_hash.unwrap_or(ScriptHash(0)).0,
+            &job.workspace_id
+        )
+        .execute(db)
+        .await?;
     } else {
-        let err = match status {
-            Ok(_) => {
-                let s = format!(
-                    "Error during execution of the script\nlast 5 logs lines:\n{}",
-                    logs.lines()
-                        .skip(logs.lines().count().max(5) - 5)
-                        .join("\n")
-                );
-                logs.push_str("\n\n--- ERROR ---\n");
-                s
-            }
-            Err(err) => format!("error before termination: {err}"),
-        };
-        Err(Error::ExecutionErr(err))
-    }
+        sqlx::query!(
+            "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3",
+            &logs.clone(),
+            &job.script_hash.unwrap_or(ScriptHash(0)).0,
+            &job.workspace_id
+        )
+        .execute(db)
+        .await?;
+    })
+}
+
+async fn get_reserved_variables(
+    job: &QueuedJob,
+    token: String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+) -> Result<HashMap<String, String>, Error> {
+    let flow_path = if let Some(uuid) = job.parent_job {
+        sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid)
+            .fetch_optional(db)
+            .await?
+            .flatten()
+    } else {
+        None
+    };
+    let variables = variables::get_reserved_variables(
+        &job.workspace_id,
+        &token,
+        &get_email_from_username(&job.created_by, db)
+            .await?
+            .unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
+        &job.created_by,
+        &job.id.to_string(),
+        &job.permissioned_as,
+        job.script_path.clone(),
+        flow_path,
+        job.schedule_path.clone(),
+    );
+    Ok(variables
+        .into_iter()
+        .map(|rv| (rv.name, rv.value))
+        .collect())
 }
 
 async fn handle_child(
@@ -621,9 +937,10 @@ async fn handle_child(
                 let canceled = sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", id)
                     .fetch_one(db)
                     .await
-                    .map_err(|_| tracing::error!("error getting canceled for id {}", id));
+                    .map_err(|e| tracing::error!("error getting canceled for id {}: {e}", id))
+                    .unwrap_or(false);
 
-                if canceled.unwrap_or(false) {
+                if canceled {
                     tracing::info!("killed after cancel: {}", job.id);
                     done.store(true, Ordering::Relaxed);
                 }
@@ -702,17 +1019,17 @@ pub async fn restart_zombie_jobs_periodically(
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) {
     loop {
-        let restarted = sqlx::query_scalar!(
-            "UPDATE queue SET running = false WHERE last_ping < $1 RETURNING id",
-            chrono::Utc::now() - chrono::Duration::seconds(timeout as i64 * 2)
+        let restarted = sqlx::query!(
+            "UPDATE queue SET running = false WHERE last_ping < $1 and running = true RETURNING id, workspace_id",
+            chrono::Utc::now() - chrono::Duration::seconds(timeout as i64 * 5)
         )
         .fetch_all(db)
         .await
         .ok()
         .unwrap_or_else(|| vec![]);
 
-        if restarted.len() > 0 {
-            tracing::info!("restarted zombie jobs {restarted:?}");
+        for r in restarted {
+            tracing::info!("restarted zombie job {} {}", r.id, r.workspace_id);
         }
 
         tokio::select! {

backend/src/worker_flow.rs

@@ -0,0 +1,604 @@
+use std::collections::HashMap;
+
+use crate::{
+    db::DB,
+    error::{self, Error},
+    flows::{FlowModuleValue, FlowValue, InputTransform},
+    jobs::{
+        add_completed_job, add_completed_job_error, get_queued_job, postprocess_queued_job, push,
+        script_path_to_payload, JobPayload, QueuedJob,
+    },
+    js_eval::{eval_timeout, EvalCreds},
+    users::create_token_for_owner,
+};
+use async_recursion::async_recursion;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Map, Value};
+use tracing::instrument;
+use uuid::Uuid;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct FlowStatus {
+    pub step: i32,
+    pub modules: Vec<FlowStatusModule>,
+    pub failure_module: FlowStatusModule,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Iterator {
+    pub index: u8,
+    pub itered: Vec<Value>,
+    pub args: Map<String, Value>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(tag = "type")]
+pub enum FlowStatusModule {
+    WaitingForPriorSteps,
+    WaitingForEvent { event: String },
+    WaitingForExecutor { job: Uuid },
+    InProgress { job: Uuid, iterator: Option<Iterator>, forloop_jobs: Option<Vec<Uuid>> },
+    Success { job: Uuid, forloop_jobs: Option<Vec<Uuid>> },
+    Failure { job: Uuid, forloop_jobs: Option<Vec<Uuid>> },
+}
+
+#[async_recursion]
+#[instrument(level = "trace", skip_all)]
+pub async fn update_flow_status_after_job_completion(
+    db: &DB,
+    job: &QueuedJob,
+    success: bool,
+    result: Option<Map<String, Value>>,
+) -> error::Result<()> {
+    tracing::debug!("HANDLE FLOW: {job:?} {success} {result:?}");
+
+    let mut tx = db.begin().await?;
+
+    let w_id = &job.workspace_id;
+
+    let flow = job
+        .parent_job
+        .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?;
+
+    let old_status_json = sqlx::query_scalar!(
+        "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
+        flow,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "fetching flow status {flow} while reporting {success} {result:?}: {e}"
+        ))
+    })?
+    .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?;
+
+    let old_status = serde_json::from_value::<FlowStatus>(old_status_json)
+        .ok()
+        .ok_or_else(|| {
+            Error::InternalErr(format!("requiring status to be parsabled as FlowStatus"))
+        })?;
+
+    let skip_loop_failures = skip_loop_failures(flow, old_status.step, &mut tx)
+        .await?
+        .unwrap_or(false);
+
+    let (step_counter, new_status) = match &old_status.modules[old_status.step as usize] {
+        module_status @ FlowStatusModule::InProgress {
+            iterator: Some(Iterator { index, itered, .. }),
+            ..
+        } if (index.to_owned() as usize) < itered.len() - 1 && (success || skip_loop_failures) => {
+            (old_status.step, module_status.clone())
+        }
+        module_status @ _ => {
+            let forloop_jobs = match module_status {
+                FlowStatusModule::InProgress { forloop_jobs: Some(jobs), .. } => Some(jobs.clone()),
+                _ => None,
+            };
+            let new_status = if success || (forloop_jobs.is_some() && skip_loop_failures) {
+                FlowStatusModule::Success { job: job.id, forloop_jobs }
+            } else {
+                FlowStatusModule::Failure { job: job.id, forloop_jobs }
+            };
+            (old_status.step + 1, new_status)
+        }
+    };
+
+    let last_step = step_counter as usize == old_status.modules.len();
+
+    tracing::debug!(
+        "old status: {:#?}\n{:#?}\n{last_step}",
+        old_status,
+        new_status
+    );
+
+    let prev_step = old_status.step;
+    let (stop_early_expr, skip_if_stop_early) =
+        sqlx::query_as::<_, (Option<String>, Option<bool>)>(&format!(
+            "UPDATE queue
+            SET 
+                flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {prev_step}}}', $1), \
+             '{{\"step\"}}', $2)
+            WHERE id = $3
+            RETURNING 
+                (raw_flow->'modules'->{prev_step}->>'stop_after_if_expr'), 
+                (raw_flow->'modules'->{prev_step}->>'skip_if_stopped')::bool",
+        ))
+        .bind(serde_json::json!(new_status))
+        .bind(serde_json::json!(step_counter))
+        .bind(flow)
+        .fetch_one(&mut tx)
+        .await
+        .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?;
+
+    tracing::debug!("UPDATE: {:?}", new_status);
+
+    let flow_job = get_queued_job(flow, w_id, &mut tx)
+        .await?
+        .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
+    tx.commit().await?;
+
+    let stop_early = success
+        && if let Some(expr) = stop_early_expr {
+            compute_stop_early(expr, result.clone()).await?
+        } else {
+            false
+        };
+
+    let done = if !(success || skip_loop_failures) || last_step || stop_early {
+        let result = match new_status {
+            FlowStatusModule::Success { forloop_jobs: Some(jobs), .. } => {
+                use futures::TryStreamExt;
+                let results = sqlx::query_as(
+                    "
+                  SELECT result
+                    FROM completed_job
+                   WHERE id = ANY($1)
+                     AND workspace_id = $2
+                     AND success = true
+                ORDER BY args->>'_index'
+                    ",
+                )
+                .bind(jobs.as_slice())
+                .bind(w_id)
+                .fetch(db)
+                .map_ok(|(v,)| v)
+                .try_collect::<Vec<serde_json::Value>>()
+                .await?;
+                let mut results_map = serde_json::Map::new();
+                results_map.insert("res1".to_string(), serde_json::json!(results));
+                Some(results_map)
+            }
+            _ => result.clone(),
+        };
+
+        let logs = if stop_early {
+            "Flow job stopped early".to_string()
+        } else {
+            "Flow job completed".to_string()
+        };
+        tracing::debug!("{skip_if_stop_early:?}");
+        add_completed_job(
+            db,
+            &flow_job,
+            success,
+            stop_early && skip_if_stop_early.unwrap_or(false),
+            result.clone(),
+            logs,
+        )
+        .await?;
+        true
+    } else {
+        match handle_flow(&flow_job, db, result.clone()).await {
+            Err(err) => {
+                let _ = add_completed_job_error(
+                    db,
+                    &flow_job,
+                    "Unexpected error during flow chaining:\n".to_string(),
+                    err,
+                )
+                .await;
+                true
+            }
+            Ok(_) => false,
+        }
+    };
+
+    if done {
+        postprocess_queued_job(
+            flow_job.is_flow_step,
+            flow_job.schedule_path.clone(),
+            flow_job.script_path.clone(),
+            &w_id,
+            flow,
+            db,
+        )
+        .await?;
+
+        if flow_job.parent_job.is_some() {
+            return Ok(
+                update_flow_status_after_job_completion(db, &flow_job, success, result).await?,
+            );
+        }
+    }
+
+    Ok(())
+}
+
+async fn skip_loop_failures<'c>(
+    flow: Uuid,
+    step: i32,
+    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+) -> Result<Option<bool>, Error> {
+    sqlx::query_as(
+        "
+    SELECT (raw_flow->'modules'->$1->'value'->>'skip_failures')::bool
+      FROM queue
+     WHERE id = $2
+        ",
+    )
+    .bind(step)
+    .bind(flow)
+    .fetch_one(tx)
+    .await
+    .map(|(v,)| v)
+    .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e}")))
+}
+
+async fn compute_stop_early(
+    expr: String,
+    result: Option<Map<String, Value>>,
+) -> error::Result<bool> {
+    let result = serde_json::Value::Object(result.clone().unwrap_or_else(|| Map::new()));
+    match eval_timeout(expr, [("result".to_string(), result)].into(), None, vec![]).await? {
+        serde_json::Value::Bool(true) => Ok(true),
+        serde_json::Value::Bool(false) => Ok(false),
+        a @ _ => Err(Error::ExecutionErr(format!(
+            "Expected a boolean value, found: {a:?}"
+        ))),
+    }
+}
+
+pub fn init_flow_status(f: &FlowValue) -> FlowStatus {
+    FlowStatus {
+        step: 0,
+        modules: vec![FlowStatusModule::WaitingForPriorSteps; f.modules.len()],
+        failure_module: FlowStatusModule::WaitingForPriorSteps,
+    }
+}
+
+pub async fn update_flow_status_in_progress(
+    db: &DB,
+    w_id: &str,
+    flow: Uuid,
+    job_in_progress: Uuid,
+) -> error::Result<()> {
+    let step = get_step_of_flow_status(db, flow).await?;
+    sqlx::query(&format!(
+        "UPDATE queue
+            SET flow_status = jsonb_set(flow_status, '{{modules, {step}, job}}', $1)
+            WHERE id = $2 AND workspace_id = $3",
+    ))
+    .bind(json!(job_in_progress.to_string()))
+    .bind(flow)
+    .bind(w_id)
+    .execute(db)
+    .await?;
+    Ok(())
+}
+
+#[instrument(level = "trace", skip_all)]
+pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<i32> {
+    let r = sqlx::query_scalar!(
+        "SELECT (flow_status->'step')::integer FROM queue WHERE id = $1",
+        id
+    )
+    .fetch_one(db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e}")))?
+    .ok_or_else(|| Error::InternalErr(format!("not found step")))?;
+    Ok(r)
+}
+
+#[instrument(level = "trace", skip_all)]
+async fn transform_input(
+    flow_args: &Option<serde_json::Value>,
+    last_result: Option<Map<String, serde_json::Value>>,
+    input_transform: &HashMap<String, InputTransform>,
+    workspace: &str,
+    token: &str,
+    steps: Vec<String>,
+) -> anyhow::Result<Option<Map<String, serde_json::Value>>> {
+    let mut mapped = serde_json::Map::new();
+
+    for (key, val) in input_transform.into_iter() {
+        match val {
+            InputTransform::Static { value } => {
+                mapped.insert(key.to_string(), value.to_owned());
+                ()
+            }
+            _ => (),
+        };
+    }
+
+    for (key, val) in input_transform.into_iter() {
+        match val {
+            InputTransform::Static { value: _ } => (),
+            InputTransform::Javascript { expr } => {
+                let previous_result =
+                    serde_json::Value::Object(last_result.clone().unwrap_or_else(|| Map::new()));
+                let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
+                let v = eval_timeout(
+                    expr.to_string(),
+                    vec![
+                        ("params".to_string(), serde_json::json!(mapped)),
+                        ("previous_result".to_string(), previous_result),
+                        ("flow_input".to_string(), flow_input),
+                    ],
+                    Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
+                    steps.clone(),
+                )
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!(
+                        "Error during isolated evaluation of expression `{expr}`:\n{e}"
+                    ))
+                })?;
+                mapped.insert(key.to_string(), v);
+                ()
+            }
+        }
+    }
+
+    Ok(Some(mapped))
+}
+
+#[instrument(level = "trace", skip_all)]
+pub async fn handle_flow(
+    flow_job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    last_result: Option<Map<String, serde_json::Value>>,
+) -> anyhow::Result<()> {
+    let value = flow_job
+        .raw_flow
+        .as_ref()
+        .ok_or_else(|| Error::InternalErr(format!("requiring a raw flow value")))?
+        .to_owned();
+    let flow = serde_json::from_value::<FlowValue>(value.to_owned())?;
+
+    if flow.modules.len() == 0 {
+        Err(Error::BadRequest(format!(
+            "A flow needs at least one module to run"
+        )))?;
+    }
+
+    push_next_flow_job(
+        flow_job,
+        flow,
+        flow_job.schedule_path.clone(),
+        db,
+        last_result,
+    )
+    .await?;
+    Ok(())
+}
+
+#[instrument(level = "trace", skip_all)]
+async fn push_next_flow_job(
+    flow_job: &QueuedJob,
+    flow: FlowValue,
+    schedule_path: Option<String>,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    last_result: Option<Map<String, serde_json::Value>>,
+) -> anyhow::Result<()> {
+    let flow_status_json = flow_job.flow_status.as_ref().ok_or_else(|| {
+        Error::InternalErr(format!("not found status for flow job {:?}", flow_job.id))
+    })?;
+    let status = serde_json::from_value::<FlowStatus>(flow_status_json.to_owned())?;
+    let i = status.step as usize;
+
+    if flow.modules.len() > i {
+        let module = &flow.modules[i];
+        let mut tx = db.begin().await?;
+        let job_payload = match &module.value {
+            FlowModuleValue::Script { path: script_path } => {
+                script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
+            }
+            FlowModuleValue::RawScript(raw_code) => {
+                let mut raw_code = raw_code.clone();
+                if raw_code.path.is_none() {
+                    raw_code.path = Some(format!(
+                        "{}/{i}",
+                        flow_job
+                            .script_path
+                            .as_ref()
+                            .unwrap_or(&"NO_FLOW_PATH".to_owned())
+                    ));
+                }
+                JobPayload::Code(raw_code)
+            }
+            FlowModuleValue::ForloopFlow { value, .. } => JobPayload::RawFlow {
+                value: *(*value).to_owned(),
+                path: Some(format!(
+                    "{}/{i}",
+                    flow_job
+                        .script_path
+                        .as_ref()
+                        .unwrap_or(&"NO_FLOW_PATH".to_owned())
+                )),
+            },
+            a @ _ => {
+                tracing::info!("Unrecognized module values {:?}", a);
+                Err(Error::BadRequest(format!(
+                    "Unrecognized module values {:?}",
+                    a
+                )))?
+            }
+        };
+
+        let token = create_token_for_owner(
+            &db,
+            &flow_job.workspace_id,
+            &flow_job.permissioned_as,
+            crate::users::NewToken {
+                label: Some("transform-input".to_string()),
+                expiration: Some(chrono::Utc::now() + chrono::Duration::seconds(10)),
+            },
+            &flow_job.created_by,
+        )
+        .await?;
+        let input_transform = module.input_transform.clone();
+
+        tracing::debug!(
+            "PUSH: module: {:#?}, status: {:#?}",
+            module.value,
+            status.modules[i]
+        );
+        let (forloop_args, forloop_iterator) = match &module.value {
+            FlowModuleValue::ForloopFlow { iterator, .. } => match &status.modules[i] {
+                FlowStatusModule::WaitingForPriorSteps { .. } => {
+                    let itered = match iterator {
+                        InputTransform::Static { value } => value.clone(),
+                        InputTransform::Javascript { expr } => {
+                            let result = serde_json::Value::Object(
+                                last_result.clone().unwrap_or_else(|| Map::new()),
+                            );
+                            eval_timeout(
+                                expr.to_string(),
+                                [("result".to_string(), result)].into(),
+                                None,
+                                vec![],
+                            )
+                            .await?
+                        }
+                    };
+
+                    let mut args = last_result.clone().unwrap_or_else(Map::new);
+
+                    args.insert(
+                        "_index".to_string(),
+                        serde_json::Value::Number(serde_json::Number::from(0)),
+                    );
+                    args.insert("_value".to_string(), itered[0].clone());
+                    match itered {
+                        serde_json::Value::Array(arr) => (Some(args), Some((0 as u8, arr, vec![]))),
+                        a @ _ => Err(Error::BadRequest(format!(
+                            "Expected an array value, found: {:?}",
+                            a
+                        )))?,
+                    }
+                }
+                FlowStatusModule::InProgress {
+                    iterator: Some(Iterator { index, itered, args }),
+                    forloop_jobs: Some(forloop_jobs),
+                    ..
+                } if index.to_owned() + 1 < itered.len() as u8 => {
+                    let mut args = args.clone();
+                    let nindex = index.to_owned() + 1;
+                    args.insert(
+                        "_index".to_string(),
+                        serde_json::Value::Number(serde_json::Number::from(nindex.to_owned())),
+                    );
+                    args.insert(
+                        "_value".to_string(),
+                        itered[nindex.to_owned() as usize].clone(),
+                    );
+                    (
+                        Some(args),
+                        Some((nindex, itered.clone(), forloop_jobs.clone())),
+                    )
+                }
+                a @ _ => Err(Error::BadRequest(format!(
+                    "Unrecognized module status for ForloopFlow {:?}",
+                    a
+                )))?,
+            },
+            _ => (None, None),
+        };
+
+        let args = if forloop_args.is_some() {
+            let mut args = forloop_args.unwrap();
+            if let Some(flow_args) = &flow_job.args {
+                match flow_args {
+                    serde_json::Value::Object(obj) => {
+                        for (k, v) in obj {
+                            args.insert(k.to_string(), v.clone());
+                        }
+                    }
+                    _ => {
+                        (Err(Error::BadRequest(format!(
+                            "Expected an object value, found: {:?}",
+                            flow_args
+                        ))))?
+                    }
+                }
+            }
+            Some(args)
+        } else {
+            let steps = status
+                .modules
+                .into_iter()
+                .map(|x| match x {
+                    FlowStatusModule::Success { job, forloop_jobs: _ } => job.to_string(),
+                    _ => "invalid step status".to_string(),
+                })
+                .collect();
+
+            let transformed = transform_input(
+                &flow_job.args,
+                last_result.clone(),
+                &input_transform,
+                &flow_job.workspace_id,
+                &token,
+                steps,
+            )
+            .await?;
+
+            transformed
+        };
+
+        let (uuid, mut tx) = push(
+            tx,
+            &flow_job.workspace_id,
+            job_payload,
+            args.clone(),
+            &flow_job.created_by,
+            flow_job.permissioned_as.to_owned(),
+            None,
+            schedule_path,
+            Some(flow_job.id),
+            true,
+        )
+        .await?;
+
+        let new_status = if let Some((index, itered, mut forloop_jobs)) = forloop_iterator {
+            forloop_jobs.push(uuid.to_owned());
+            serde_json::json!(FlowStatusModule::InProgress {
+                job: uuid,
+                iterator: Some(Iterator {
+                    index: index,
+                    itered: itered,
+                    args: args.unwrap_or_else(|| Map::new()),
+                }),
+                forloop_jobs: Some(forloop_jobs),
+            })
+        } else {
+            serde_json::json!(FlowStatusModule::WaitingForExecutor { job: uuid })
+        };
+
+        sqlx::query(&format!(
+            "UPDATE queue
+            SET 
+                flow_status = jsonb_set(flow_status, '{{modules, {}}}', $1)
+            WHERE id = $2",
+            i
+        ))
+        .bind(new_status)
+        .bind(flow_job.id)
+        .execute(&mut tx)
+        .await?;
+        tx.commit().await?;
+    }
+    Ok(())
+}

backend/src/worker_ping.rs

@@ -1,5 +1,6 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.

backend/src/workspaces.rs

@@ -1,20 +1,33 @@
 /*
- * Author & Copyright: Ruben Fiszel 2021
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
  * This file and its contents are licensed under the AGPLv3 License.
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
 
 use crate::{
+    audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{Error, JsonResult, Result},
-    users::{Authed, WorkspaceInvite}, utils::{require_admin, require_super_admin, Pagination}, audit::{audit_log, ActionKind}, scripts::{Script, Schema}, resources::{Resource, ResourceType}, flow::Flow, variables::ListableVariable,
+    flows::Flow,
+    resources::{Resource, ResourceType},
+    scripts::{Schema, Script},
+    users::{Authed, WorkspaceInvite},
+    utils::{require_admin, require_super_admin, Pagination},
+    variables::ListableVariable,
+};
+use axum::{
+    body::StreamBody,
+    extract::{Extension, Path, Query},
+    response::IntoResponse,
+    routing::{delete, get, post},
+    Json, Router,
 };
-use axum::{extract::{Extension, Path, Query}, routing::{get, post, delete}, Json, Router, response::{IntoResponse}, body::StreamBody};
 
-use hyper::{StatusCode, header};
+use hyper::{header, StatusCode};
 use serde::{Deserialize, Serialize};
-use sqlx::{FromRow};
+use sqlx::FromRow;
 use tempfile::TempDir;
 use tokio::fs::File;
 use tokio_util::io::ReaderStream;
@@ -29,22 +42,16 @@ pub fn workspaced_service() -> Router {
         .route("/get_settings", get(get_settings))
         .route("/edit_slack_command", post(edit_slack_command))
         .route("/tarball", get(tarball_workspace))
-
-
-
 }
 
 pub fn global_service() -> Router {
     Router::new()
-    .route("/list_as_superadmin", get(list_workspaces_as_super_admin))
-    .route("/list", get(list_workspaces))
-    .route("/users", get(user_workspaces))
-    .route("/create", post(create_workspace))
-    .route("/exists", post(exists_workspace))
-    .route("/validate_username", post(validate_username))
-    .route("/validate_id", post(validate_id))
-
-
+        .route("/list_as_superadmin", get(list_workspaces_as_super_admin))
+        .route("/list", get(list_workspaces))
+        .route("/users", get(user_workspaces))
+        .route("/create", post(create_workspace))
+        .route("/exists", post(exists_workspace))
+        .route("/exists_username", post(exists_username))
 }
 
 #[derive(FromRow, Serialize)]
@@ -53,7 +60,8 @@ struct Workspace {
     name: String,
     owner: String,
     domain: Option<String>,
-    deleted: bool
+    deleted: bool,
+    premium: bool,
 }
 
 #[derive(FromRow, Serialize, Debug)]
@@ -61,20 +69,18 @@ pub struct WorkspaceSettings {
     pub workspace_id: String,
     pub slack_team_id: Option<String>,
     pub slack_name: Option<String>,
-    pub slack_command_script: Option<String>
+    pub slack_command_script: Option<String>,
 }
 
-
 #[derive(sqlx::Type, Serialize, Deserialize, Debug)]
 #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")]
 pub enum WorkspaceKeyKind {
-    Cloud
+    Cloud,
 }
 
-
 #[derive(Deserialize)]
 struct EditCommandScript {
-    slack_command_script: Option<String>
+    slack_command_script: Option<String>,
 }
 #[derive(Deserialize)]
 struct CreateWorkspace {
@@ -84,15 +90,13 @@ struct CreateWorkspace {
     domain: Option<String>,
 }
 
-
 #[derive(Deserialize)]
 struct EditWorkspace {
     name: String,
     owner: String,
-    domain: Option<String>
+    domain: Option<String>,
 }
 
-
 #[derive(Serialize)]
 struct WorkspaceList {
     pub email: String,
@@ -106,7 +110,6 @@ struct UserWorkspace {
     pub username: String,
 }
 
-
 #[derive(Deserialize)]
 struct WorkspaceId {
     pub id: String,
@@ -118,14 +121,12 @@ struct ValidateUsername {
     pub username: String,
 }
 
-
 #[derive(Deserialize)]
 pub struct NewWorkspaceInvite {
     pub email: String,
     pub is_admin: bool,
 }
 
-
 async fn list_pending_invites(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -133,26 +134,30 @@ async fn list_pending_invites(
 ) -> JsonResult<Vec<WorkspaceInvite>> {
     require_admin(authed.is_admin, &authed.username)?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as!(WorkspaceInvite, "SELECT * from workspace_invite WHERE workspace_id = $1", w_id)
-        .fetch_all(&mut tx)
-        .await?;
+    let rows = sqlx::query_as!(
+        WorkspaceInvite,
+        "SELECT * from workspace_invite WHERE workspace_id = $1",
+        w_id
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
-
 async fn exists_workspace(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
-    Json(WorkspaceId { id }): Json<WorkspaceId>
+    Json(WorkspaceId { id }): Json<WorkspaceId>,
 ) -> JsonResult<bool> {
     let mut tx = user_db.begin(&authed).await?;
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)", 
-        id)
-        .fetch_one(&mut tx)
-        .await?
-        .unwrap_or(false);
+        "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)",
+        id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
     tx.commit().await?;
     Ok(Json(exists))
 }
@@ -163,16 +168,17 @@ async fn list_workspaces(
 ) -> JsonResult<Vec<Workspace>> {
     let mut tx = user_db.begin(&authed).await?;
     let workspaces = sqlx::query_as!(
-        Workspace, 
-        "SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false", 
-        authed.email.as_ref())
-        .fetch_all(&mut tx)
-        .await?;
+        Workspace,
+        "SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \
+         usr.email = $1 AND deleted = false",
+        authed.email.as_ref()
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(workspaces))
 }
 
-
 async fn get_settings(
     authed: Authed,
     Path(w_id): Path<String>,
@@ -180,11 +186,14 @@ async fn get_settings(
 ) -> JsonResult<WorkspaceSettings> {
     let mut tx = user_db.begin(&authed).await?;
     let settings = sqlx::query_as!(
-        WorkspaceSettings, 
-        "SELECT * FROM workspace_settings WHERE workspace_id = $1", 
-        &w_id)
-        .fetch_one(&mut tx)
-        .await?;
+        WorkspaceSettings,
+        "SELECT * FROM workspace_settings WHERE workspace_id = $1",
+        &w_id
+    )
+    .fetch_one(&mut tx)
+    .await
+    .map_err(|e| Error::InternalErr(format!("getting settings: {e}")))?;
+
     tx.commit().await?;
     Ok(Json(settings))
 }
@@ -194,7 +203,7 @@ async fn edit_slack_command(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Authed { is_admin, username, .. }: Authed,
-    Json(es): Json<EditCommandScript>
+    Json(es): Json<EditCommandScript>,
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
@@ -207,15 +216,21 @@ async fn edit_slack_command(
     .await?;
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &authed.username,
         "workspaces.edit_command_script",
         ActionKind::Update,
         &w_id,
-Some(&authed.email.unwrap()),
-Some([
-            ("script", es.slack_command_script.unwrap_or("NO_SCRIPT".to_string()).as_str())
-        ].into()),
+        Some(&authed.email.unwrap()),
+        Some(
+            [(
+                "script",
+                es.slack_command_script
+                    .unwrap_or("NO_SCRIPT".to_string())
+                    .as_str(),
+            )]
+            .into(),
+        ),
     )
     .await?;
     tx.commit().await?;
@@ -223,7 +238,6 @@ Some([
     Ok(format!("Edit command script {}", &w_id))
 }
 
-
 async fn list_workspaces_as_super_admin(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -235,10 +249,13 @@ async fn list_workspaces_as_super_admin(
     let (per_page, offset) = crate::utils::paginate(pagination);
 
     let workspaces = sqlx::query_as!(
-        Workspace, 
-        "SELECT * FROM workspace LIMIT $1 OFFSET $2", per_page as i32, offset as i32)
-        .fetch_all(&mut tx)
-        .await?;
+        Workspace,
+        "SELECT * FROM workspace LIMIT $1 OFFSET $2",
+        per_page as i32,
+        offset as i32
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(workspaces))
 }
@@ -252,12 +269,14 @@ async fn user_workspaces(
         .map_err(|x| Error::NotAuthorized(x.to_string()))?;
     let mut tx = db.begin().await?;
     let workspaces = sqlx::query_as!(
-        UserWorkspace, 
-    "SELECT workspace.id, workspace.name, usr.username
-     FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false", 
-     email)
-        .fetch_all(&mut tx)
-        .await?;
+        UserWorkspace,
+        "SELECT workspace.id, workspace.name, usr.username
+     FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = \
+         false",
+        email
+    )
+    .fetch_all(&mut tx)
+    .await?;
     tx.commit().await?;
     Ok(Json(WorkspaceList { email, workspaces }))
 }
@@ -265,10 +284,10 @@ async fn user_workspaces(
 async fn create_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Json(nw): Json<CreateWorkspace>
+    Json(nw): Json<CreateWorkspace>,
 ) -> Result<String> {
     if &nw.username == "bot" {
-        return Err(Error::BadRequest("bot is a reserved username".to_string()))
+        return Err(Error::BadRequest("bot is a reserved username".to_string()));
     }
     let mut tx = db.begin().await?;
     sqlx::query!(
@@ -295,7 +314,8 @@ async fn create_workspace(
         "INSERT INTO workspace_key
             (workspace_id, kind, key)
             VALUES ($1, 'cloud', $2)",
-        nw.id, &key
+        nw.id,
+        &key
     )
     .execute(&mut tx)
     .await?;
@@ -307,7 +327,7 @@ async fn create_workspace(
             VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), 
                 ($3, 'g/all/not_secret', $4, false, 'This item is not secret')",
         nw.id,
-        crate::variables::encrypt(&mc, "pretty secret value".to_string()),
+        crate::variables::encrypt(&mc, "pretty secret value"),
         nw.id,
         "finland does not actually exist",
     )
@@ -343,17 +363,16 @@ async fn create_workspace(
     .await?;
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &authed.username,
         "workspaces.create",
         ActionKind::Create,
         &nw.id,
-        Some(&authed.email.unwrap()),
+        Some(nw.name.as_str()),
         None,
     )
     .await?;
     tx.commit().await?;
-    
 
     Ok(format!("Created workspace {}", &nw.id))
 }
@@ -363,7 +382,7 @@ async fn edit_workspace(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Authed { is_admin, username, .. }: Authed,
-    Json(ew): Json<EditWorkspace>
+    Json(ew): Json<EditWorkspace>,
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
@@ -378,15 +397,19 @@ async fn edit_workspace(
     .await?;
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &authed.username,
         "workspaces.update",
         ActionKind::Update,
         &w_id,
-Some(&authed.email.unwrap()),
-Some([
-            ("domain", ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str())
-        ].into()),
+        Some(&authed.email.unwrap()),
+        Some(
+            [(
+                "domain",
+                ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str(),
+            )]
+            .into(),
+        ),
     )
     .await?;
     tx.commit().await?;
@@ -401,21 +424,18 @@ async fn delete_workspace(
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
-    sqlx::query!(
-        "UPDATE workspace SET deleted = true WHERE id = $1",
-        &w_id
-    )
-    .execute(&mut tx)
-    .await?;
+    sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", &w_id)
+        .execute(&mut tx)
+        .await?;
 
     audit_log(
-         &mut tx,
+        &mut tx,
         &username,
         "workspaces.delete",
         ActionKind::Update,
         &w_id,
-Some(&email.unwrap_or("noemail".to_string())),
-None,
+        Some(&email.unwrap_or("noemail".to_string())),
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -423,14 +443,12 @@ None,
     Ok(format!("Deleted workspace {}", &w_id))
 }
 
-
 async fn invite_user(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
-
     require_admin(is_admin, &username)?;
 
     let mut tx = db.begin().await?;
@@ -454,14 +472,12 @@ async fn invite_user(
     ))
 }
 
-
 async fn delete_invite(
     Authed { username, is_admin, .. }: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(nu): Json<NewWorkspaceInvite>,
 ) -> Result<(StatusCode, String)> {
-
     require_admin(is_admin, &username)?;
 
     let mut tx = db.begin().await?;
@@ -484,11 +500,10 @@ async fn delete_invite(
     ))
 }
 
-async fn validate_username(
+async fn exists_username(
     Extension(db): Extension<DB>,
     Json(vu): Json<ValidateUsername>,
 ) -> Result<String> {
-
     let exists = sqlx::query_scalar!(
         "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)",
         vu.username,
@@ -499,88 +514,79 @@ async fn validate_username(
     .unwrap_or(true);
 
     if exists {
-        return Err(Error::BadRequest("username already taken".to_string()))
+        return Err(Error::BadRequest("username already taken".to_string()));
     }
 
     Ok("valid username".to_string())
 }
 
-
-async fn validate_id(
-    Extension(db): Extension<DB>,
-    Json(wi): Json<WorkspaceId>,
-) -> Result<String> {
-
-    let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)",
-        wi.id
-    )
-    .fetch_one(&db)
-    .await?
-    .unwrap_or(true);
-
-    if exists {
-        return Err(Error::BadRequest("id already taken".to_string()))
-    }
-
-    Ok("valid workspace".to_string())
-}
-
 #[derive(Serialize)]
 struct ScriptMetadata {
     summary: String,
     description: String,
     schema: Option<Schema>,
     is_template: bool,
-    lock: Vec<String>
+    lock: Vec<String>,
 }
 
 async fn tarball_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-) ->  Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
+) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
     let tmp_dir = TempDir::new_in(".")?;
-    
+
     let name = format!("windmill-{w_id}.tar");
     let file_path = tmp_dir.path().join(&name);
     let file = File::create(&file_path).await?;
     let mut a = tokio_tar::Builder::new(file);
-     
 
     {
         let scripts = sqlx::query_as::<_, Script>(
-        "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
-            AND created_at = (select max(created_at) from script where path = o.path AND workspace_id = $1)"
-    )
-    .bind(&w_id)
-    .fetch_all(&db)
-    .await?;
-
-    for script in scripts {
-        write_to_archive(script.content, format!("scripts/{}.py", script.path), &mut a).await?;
-
-        let lock = script.lock.unwrap_or_else(|| "".to_string())
-        .lines()
-        .map(|x| x.to_string())
-        .collect();
-        let metadata = ScriptMetadata { 
-            summary: script.summary, description: script.description, schema: script.schema, is_template: script.is_template, lock };
-        let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
-        write_to_archive(metadata_str, format!("scripts/{}.json", script.path), &mut a).await?;
-
-    };
-}
-
-
+            "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
+            AND created_at = (select max(created_at) from script where path = o.path AND \
+             workspace_id = $1)",
+        )
+        .bind(&w_id)
+        .fetch_all(&db)
+        .await?;
 
+        for script in scripts {
+            write_to_archive(
+                script.content,
+                format!("scripts/{}.py", script.path),
+                &mut a,
+            )
+            .await?;
 
+            let lock = script
+                .lock
+                .unwrap_or_else(|| "".to_string())
+                .lines()
+                .map(|x| x.to_string())
+                .collect();
+            let metadata = ScriptMetadata {
+                summary: script.summary,
+                description: script.description,
+                schema: script.schema,
+                is_template: script.is_template,
+                lock,
+            };
+            let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
+            write_to_archive(
+                metadata_str,
+                format!("scripts/{}.json", script.path),
+                &mut a,
+            )
+            .await?;
+        }
+    }
 
     {
-
-        let resources = sqlx::query_as!(Resource,
+        let resources = sqlx::query_as!(
+            Resource,
             "SELECT * FROM resource WHERE workspace_id = $1",
             &w_id
         )
@@ -589,27 +595,38 @@ async fn tarball_workspace(
 
         for resource in resources {
             let resource_str = serde_json::to_string_pretty(&resource).unwrap();
-            write_to_archive(resource_str, format!("resources/{}.json", resource.path), &mut a).await?;
+            write_to_archive(
+                resource_str,
+                format!("resources/{}.json", resource.path),
+                &mut a,
+            )
+            .await?;
         }
     }
 
     {
-        let resource_types = sqlx::query_as!(ResourceType,
-        "SELECT * FROM resource_type WHERE workspace_id = $1",
-        &w_id
-    )
-    .fetch_all(&db)
-    .await?;
+        let resource_types = sqlx::query_as!(
+            ResourceType,
+            "SELECT * FROM resource_type WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_all(&db)
+        .await?;
 
         for resource_type in resource_types {
             let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
-            write_to_archive(resource_str, format!("resource_types/{}.json", resource_type.name), &mut a).await?;
+            write_to_archive(
+                resource_str,
+                format!("resource_types/{}.json", resource_type.name),
+                &mut a,
+            )
+            .await?;
         }
     }
 
     {
         let flows = sqlx::query_as::<_, Flow>(
-            "SELECT * FROM flow WHERE workspace_id = $1 AND archived = false"
+            "SELECT * FROM flow WHERE workspace_id = $1 AND archived = false",
         )
         .bind(&w_id)
         .fetch_all(&db)
@@ -622,8 +639,8 @@ async fn tarball_workspace(
     }
 
     {
-       let variables = sqlx::query_as::<_, ListableVariable>(
-        "SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false"
+        let variables = sqlx::query_as::<_, ListableVariable>(
+            "SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false",
         )
         .bind(&w_id)
         .fetch_all(&db)
@@ -652,7 +669,11 @@ async fn tarball_workspace(
     Ok((headers, body))
 }
 
-async fn write_to_archive(content: String, path: String, a: &mut tokio_tar::Builder<File>) -> Result<()> {
+async fn write_to_archive(
+    content: String,
+    path: String,
+    a: &mut tokio_tar::Builder<File>,
+) -> Result<()> {
     let bytes = content.as_bytes();
     let mut header = tokio_tar::Header::new_gnu();
     header.set_size(bytes.len() as u64);

community/resource_types/airtable.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "airtable",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": ""
+      }
+    },
+    "required": [
+      "token"
+    ],
+    "type": "object"
+  },
+  "description": "The airtable OAuth token"
+}

community/resource_types/airtable_table.json

@@ -0,0 +1,20 @@
+{
+  "workspace_id": "starter",
+  "name": "airtable_table",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "baseId": {
+        "type": "string",
+        "description": ""
+      },
+      "tableName": {
+        "type": "string",
+        "description": ""
+      }
+    },
+    "required": [],
+    "type": "object"
+  },
+  "description": ""
+}

community/resource_types/datadog.json

@@ -0,0 +1,26 @@
+{
+  "workspace_id": "starter",
+  "name": "datadog",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "apiKey": {
+        "type": "string",
+        "description": ""
+      },
+      "appKey": {
+        "type": "string",
+        "description": ""
+      },
+      "apiBase": {
+        "type": "string",
+        "description": ""
+      }
+    },
+    "required": [
+      "apiKey"
+    ],
+    "type": "object"
+  },
+  "description": ""
+}

community/resource_types/email_smtp.json

@@ -1,6 +1,6 @@
 {
   "workspace_id": "starter",
-  "name": "SMTP",
+  "name": "email_smtp",
   "schema": {
     "type": "object",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
@@ -28,5 +28,5 @@
       }
     }
   },
-  "description": "SMTP connection info"
-}
+  "description": "SMTP connection infos"
+}

community/resource_types/gcal.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "gcal",
+  "schema": {
+    "type": "object",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "required": [
+      "token"
+    ],
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": "The OAuth token, refreshed if necessary"
+      }
+    }
+  },
+  "description": "GCalendar OAuth credentials"
+}

community/resource_types/gdrive.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "gdrive",
+  "schema": {
+    "type": "object",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "required": [
+      "token"
+    ],
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": ""
+      }
+    }
+  },
+  "description": "GDrive OAuth credentials"
+}

community/resource_types/github.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "github",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": ""
+      }
+    },
+    "required": [
+      "token"
+    ],
+    "type": "object"
+  },
+  "description": "The github OAuth credentials"
+}