all

* feat(frontend): Add debug runs

* feat(frontend): Fix build

* feat(frontend): revert implementation

* feat(frontend): Fix error displa

* feat(frontend): Fix error display

* feat(frontend): Fix preview mode

* feat(frontend): Fix error display

.env

@@ -1,2 +1,2 @@
 DB_PASSWORD=changeme
-WM_BASE_URL=windmill.localhost
+WM_BASE_URL=localhost

.github/pull_hub_items.sh

@@ -9,6 +9,6 @@ for item in ${RT[@]}; do
     jq -r '.resource_type.schema' <<< "$body" > ./tmp
     description=$(jq -r '.resource_type.description' <<< "$body")
     description=$(echo -E $description)
-    echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
+    echo "{\"workspace_id\": \"admins\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
     rm ./tmp
 done

.github/uffizzi/caddy/Caddyfile

@@ -0,0 +1,5 @@
+localhost {
+        bind 0.0.0.0
+        reverse_proxy /ws/* http://0.0.0.0:3001
+        reverse_proxy /* http://0.0.0.0:8000
+}

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,49 @@
+version: '3.7'
+
+x-uffizzi:
+  ingress:
+    service: windmill
+    port: 8000
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: windmill
+
+  windmill:
+    image: '${WINDMILL_IMAGE}'
+    privileged: false
+    restart: unless-stopped
+    ports:
+      - 8000:8000
+    environment:
+      - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
+      - BASE_URL=http://localhost
+      - BASE_INTERNAL_URL=http://localhost:8000
+      - RUST_LOG=info
+      - NUM_WORKERS=3
+      - KEEP_JOB_DIR=false
+      - DENO_PATH=/usr/bin/deno
+      - PYTHON_PATH=/usr/local/bin/python3
+      - METRICS_ADDR=false
+    volumes:
+      - worker_dependency_cache:/tmp/windmill/cache
+
+  lsp:
+    image: '${LSP_IMAGE}'
+    restart: unless-stopped
+    ports:
+      - 3001:3001
+
+  # caddy:
+  #   image: caddy:2.5.2-alpine
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./.github/uffizzi/caddy:/etc/caddy
+  #   environment:
+  #     - BASE_URL=localhost
+
+volumes:
+  worker_dependency_cache:

.github/workflows/automerge-dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.3.4
+        uses: dependabot/fetch-metadata@v1.3.5
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Enable auto-merge for Dependabot PRs

.github/workflows/backend-test.yml

@@ -39,5 +39,5 @@ jobs:
             backend
             backend -> target
       - name: cargo test
-        timeout-minutes: 5
-        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        timeout-minutes: 10
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture

.github/workflows/change-versions.yml

@@ -12,4 +12,9 @@ jobs:
       - uses: actions/checkout@v3
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: update lockfile
+        run: cd backend && cargo generate-lockfile
       - uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/docker-image-arm.yml

@@ -0,0 +1,60 @@
+# env:
+#   REGISTRY: ghcr.io
+#   IMAGE_NAME: ${{ github.repository }}
+
+# name: Build and push arm docker image
+# on:
+#   push:
+#     branches: [main]
+#     tags: ["*"]
+
+# concurrency:
+#   group: ${{ github.ref }}-arm
+#   cancel-in-progress: true
+
+# permissions:
+#   contents: read
+#   id-token: write
+#   packages: write
+
+# jobs:
+#   publish_arm:
+#     runs-on: ubuntu-22.04
+#     steps:
+#       - uses: actions/checkout@v3
+#         with:
+#           fetch-depth: 0
+
+#       - uses: depot/setup-action@v1
+
+#       - name: Docker meta
+#         id: meta-slim-public
+#         uses: docker/metadata-action@v4
+#         with:
+#           images: |
+#             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+#           tags: |
+#             type=ref,event=branch
+#             type=ref,event=pr
+#             type=semver,pattern={{version}}
+#             type=semver,pattern={{major}}.{{minor}}
+
+#       - name: Login to registry
+#         uses: docker/login-action@v2
+#         with:
+#           registry: ${{ env.REGISTRY }}
+#           username: ${{ github.actor }}
+#           password: ${{ secrets.GITHUB_TOKEN }}
+
+#       - name: Build and push publicly
+#         uses: depot/build-push-action@v1
+#         with:
+#           context: .
+#           push: true
+#           platforms: linux/amd64,linux/arm64
+#           tags: |
+#             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+#             ${{ steps.meta-slim-public.outputs.tags }}
+#           labels: |
+#             ${{ steps.meta-slim-public.outputs.labels }}
+#             org.opencontainers.image.licenses=AGPLv3

.github/workflows/docker-image.yml

@@ -9,54 +9,78 @@ on:
   push:
     branches: [main]
     tags: ["*"]
-  pull_request:
-    types: [opened, synchronize, reopened]
 
 concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+  
 jobs:
   build:
-    runs-on: [self-hosted, new]
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Docker meta local
-        id: metalocal
+
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+      - uses: depot/setup-action@v1
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta-public
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-      - name: Build and push privately
-        uses: docker/build-push-action@v3
-        if: github.event_name == 'pull_request'
+
+      - name: Build and push publicly
+        uses: depot/build-push-action@v1
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ${{ steps.metalocal.outputs.tags }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.meta-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-public.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
+   
+  build_ee:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
 
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+      - uses: depot/setup-action@v1
 
       - name: Docker meta
-        if: github.event_name != 'pull_request'
-        id: meta
+        id: meta-ee-public
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -64,34 +88,28 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
 
       - name: Login to registry
-        if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Login to ECR
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.ECR_REGISTRY }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-      - name: Build and push publicly
-        uses: docker/build-push-action@v3
-        if: github.event_name != 'pull_request'
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
         with:
           context: .
           push: true
+          build-args: |
+            features=enterprise
+            nsjail=true
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.metalocal.outputs.tags }}
-            ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+
+
   playwright:
     runs-on: [self-hosted, new]
     needs: [build]
@@ -112,11 +130,105 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: "Docker"
-        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
         id: docker-container
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
       - name: "Playwright run"
         timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
+        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
       - name: "Clean up"
         run: docker kill ${{ steps.docker-container.outputs.id }}
         if: always()
+
+
+  publish_privately_heavy:
+    needs: [build_ee]
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta-heavy
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push privately
+        uses: docker/build-push-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileHeavy
+          tags: |
+            ${{ steps.meta-heavy.outputs.tags }}
+          labels: ${{ steps.meta-heavy.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache,mode=max
+
+  publish_privately_helm:
+    runs-on: [self-hosted, new]
+    needs: [build_ee]
+    if: github.event_name != 'pull_request'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+ 
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          
+      - name: Build and push privately
+        uses: docker/build-push-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileHelm
+          tags: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache,mode=max

.github/workflows/frontend-check.yml

@@ -0,0 +1,17 @@
+name: check frontend build
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+    paths:
+      - "frontend/**"
+jobs:
+  npm_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: "npm check"
+        timeout-minutes: 2
+        run: cd frontend && npm ci && npm run generate-backend-client && npm run check

.github/workflows/pull-hub.yml

@@ -1,20 +0,0 @@
-name: Pull Hub Items
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 */1 * *"
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  change_version:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Pull hub
-        run: ./.github/pull_hub_items.sh
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        with:
-          title: sync hub items with community
-          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -28,6 +28,9 @@ jobs:
     needs: [publish_pypi]
     runs-on: [self-hosted, new]
     steps:
+      - name: Sleep for 30 seconds waiting for pypi to update index
+        run: sleep 30s
+        shell: bash
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -71,6 +74,7 @@ jobs:
             ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
             ${{ steps.metalocal.outputs.tags }}
             ${{ steps.meta.outputs.tags }}
+            registry.uffizzi.com/windmill-lsp:60d
           labels: ${{ steps.metalocal.outputs.labels }}
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
           cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,91 @@
+name: Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+    paths:
+      - "backend/**"
+  workflow_dispatch:
+      
+jobs:
+  build-windmill:
+    name: Build and Push `windmill`
+    runs-on: ubuntu-latest
+    if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: ./
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs:
+      - build-windmill
+    outputs:
+      compose-file-cache-key: ${{ steps.hash.outputs.hash }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
+          export WINDMILL_IMAGE
+          LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
+          export LSP_IMAGE
+          envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }}
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run: echo '${{ toJSON(github.event) }}' > event.json
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview.yml

@@ -0,0 +1,102 @@
+name: Deploy Uffizzi Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            if (matchArtifact === undefined) {
+              throw TypeError('Build Artifact not found!');
+            }
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.rendered.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.rendered.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+
+  playwright:
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-uffizzi-preview
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: "Playwright run"
+        timeout-minutes: 2
+        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test

.gitignore

@@ -4,3 +4,4 @@ nohup.out
 local/
 frontend/src/routes/test.svelte
 CaddyfileRemoteMalo
+*.swp

CHANGELOG.md

@@ -1,6 +1,340 @@
 # Changelog
 
 
+## [1.60.0](https://github.com/windmill-labs/windmill/compare/v1.59.0...v1.60.0) (2023-01-11)
+
+
+### Features
+
+* add 'add user to workspace' ([a14623f](https://github.com/windmill-labs/windmill/commit/a14623feaab4a36c01d558b775a42e587a74cdc9))
+* **frontend:** Add frost to color palette ([#1084](https://github.com/windmill-labs/windmill/issues/1084)) ([8e72007](https://github.com/windmill-labs/windmill/commit/8e7200736827e8f6e593f900124b1bd1bc0bd5f2))
+
+
+### Bug Fixes
+
+* **frontend:** Keep pane resizer under open drawer ([#1089](https://github.com/windmill-labs/windmill/issues/1089)) ([cb25f88](https://github.com/windmill-labs/windmill/commit/cb25f883005b99b4ce98e8ae7b8253a8a2fedb5b))
+
+## [1.59.0](https://github.com/windmill-labs/windmill/compare/v1.58.0...v1.59.0) (2023-01-09)
+
+
+### Features
+
+* add relative imports for python scripts ([#1075](https://github.com/windmill-labs/windmill/issues/1075)) ([5347cd4](https://github.com/windmill-labs/windmill/commit/5347cd46a996b4cf48a96fbb873e4d029ca4f75f))
+
+
+### Bug Fixes
+
+* **frontend:** Iconed resource height issue ([#1073](https://github.com/windmill-labs/windmill/issues/1073)) ([a84eb9b](https://github.com/windmill-labs/windmill/commit/a84eb9b1f7e1b10c960ee1594ef476e7ba146f5e))
+
+## [1.58.0](https://github.com/windmill-labs/windmill/compare/v1.57.1...v1.58.0) (2023-01-07)
+
+
+### Features
+
+* add archive/unarchive/delete workspace ([6edf9b9](https://github.com/windmill-labs/windmill/commit/6edf9b9946d613b599cb91688c4986044caaba8d))
+* add hub support for apps ([50453ca](https://github.com/windmill-labs/windmill/commit/50453ca690dfd936474ebbf000e36ae1006b188b))
+* add min/max constraint to number + slider component ([0bcdcae](https://github.com/windmill-labs/windmill/commit/0bcdcaedcfdf7b7f76f703df3bf50d97dd389995))
+* add support for yaml format as a string format ([5204e4a](https://github.com/windmill-labs/windmill/commit/5204e4a75d74e6bb4087dee7087390f7c0388e51))
+* **frontend:** Add integration icons ([#1063](https://github.com/windmill-labs/windmill/issues/1063)) ([45acb89](https://github.com/windmill-labs/windmill/commit/45acb89f87ad78c48a1ba6abf1bd1424088b41c4))
+* **frontend:** Toggle to hide optional inputs ([#1060](https://github.com/windmill-labs/windmill/issues/1060)) ([4d6a568](https://github.com/windmill-labs/windmill/commit/4d6a568820ceb6c064dc2871085b80412e18c379))
+* **frontend:** Update app auto-refresh button ([#1062](https://github.com/windmill-labs/windmill/issues/1062)) ([34e3331](https://github.com/windmill-labs/windmill/commit/34e33319192f6d747d84fc6559853410f5d72ec8))
+
+
+### Bug Fixes
+
+* **frontend:** Remove popover hover styles ([#1064](https://github.com/windmill-labs/windmill/issues/1064)) ([76a860f](https://github.com/windmill-labs/windmill/commit/76a860fe538dadfc6691074384f92db1a331063d))
+
+## [1.57.1](https://github.com/windmill-labs/windmill/compare/v1.57.0...v1.57.1) (2023-01-02)
+
+
+### Bug Fixes
+
+* preserver order changes for flows' schema ([2c8e98a](https://github.com/windmill-labs/windmill/commit/2c8e98a9c7fe3fdd48c851c0575fdb1d87c953a9))
+* support setting undefined states ([ab0aeb0](https://github.com/windmill-labs/windmill/commit/ab0aeb0df825fb5afefbefae6739179dbbbc5f30))
+
+## [1.57.0](https://github.com/windmill-labs/windmill/compare/v1.56.1...v1.57.0) (2023-01-01)
+
+
+### Features
+
+* add a All Static Inputs module to the flow editor ([3296deb](https://github.com/windmill-labs/windmill/commit/3296debfe7940fe833d489af0a4b6609c2d53411))
+* apps can be published publicly ([be14aab](https://github.com/windmill-labs/windmill/commit/be14aab9b102ef81eccf689e00cd3cd8eae8f503))
+* **app:** Update sidebar menu ([#1050](https://github.com/windmill-labs/windmill/issues/1050)) ([faa046a](https://github.com/windmill-labs/windmill/commit/faa046a3fdc326084df93f8e57dd5c573164b91d))
+* **app:** Use consistent styles on settings pages ([#1048](https://github.com/windmill-labs/windmill/issues/1048)) ([681e2e8](https://github.com/windmill-labs/windmill/commit/681e2e824a39d9748f1aaa37f20001b5200f82bc))
+* **backend:** resume from owner directly in flow status viewer ([#1042](https://github.com/windmill-labs/windmill/issues/1042)) ([40195d4](https://github.com/windmill-labs/windmill/commit/40195d42f661d401cd9ce11ca9739f87c1a27afd))
+* **frontend:** Add customization props to radio ([#1056](https://github.com/windmill-labs/windmill/issues/1056)) ([0812f6e](https://github.com/windmill-labs/windmill/commit/0812f6efd8484e86a4f09631b28c71d17cd69627))
+* **frontend:** Fix initial component dimensions + Select select + add spinner when buttons are clicked ([#1044](https://github.com/windmill-labs/windmill/issues/1044)) ([70e7a5d](https://github.com/windmill-labs/windmill/commit/70e7a5d07542e1ac936152e434146e056a80bad4))
+* **frontend:** Properly support resource ([#1039](https://github.com/windmill-labs/windmill/issues/1039)) ([65f4e86](https://github.com/windmill-labs/windmill/commit/65f4e86a22838bd34373ce808c77a1770eeaf295))
+* **frontend:** Update tooltip and home list dropdown ([#1053](https://github.com/windmill-labs/windmill/issues/1053)) ([9d30e5f](https://github.com/windmill-labs/windmill/commit/9d30e5fa57363c4cf715f845f5268856c4aa0fb3))
+
+
+### Bug Fixes
+
+* **app:** Fix inconsistencies in list items and sidebar menus ([#1051](https://github.com/windmill-labs/windmill/issues/1051)) ([0f1b19c](https://github.com/windmill-labs/windmill/commit/0f1b19c7d3eea4f8106fed3188460678e5035812))
+* **frontend:** List item overflowing corners ([#1055](https://github.com/windmill-labs/windmill/issues/1055)) ([2fd730f](https://github.com/windmill-labs/windmill/commit/2fd730f8d2303b57f2da42354cd207dad2a410ce))
+* **frontend:** Minor fixes in editor ([#1054](https://github.com/windmill-labs/windmill/issues/1054)) ([adc84f0](https://github.com/windmill-labs/windmill/commit/adc84f06d97275b17bf77cb6c8d264ad28b0f6ce))
+* **frontend:** Static inputs overflow ([#1057](https://github.com/windmill-labs/windmill/issues/1057)) ([72aeba1](https://github.com/windmill-labs/windmill/commit/72aeba121cb694e8e96ad189b4acbfc2340bf520))
+
+## [1.56.1](https://github.com/windmill-labs/windmill/compare/v1.56.0...v1.56.1) (2022-12-23)
+
+
+### Bug Fixes
+
+* **cli:** typo in cli deps ([0614ec4](https://github.com/windmill-labs/windmill/commit/0614ec42baf3e8f1675d62ca0f143b831c2700a1))
+
+## [1.56.0](https://github.com/windmill-labs/windmill/compare/v1.55.0...v1.56.0) (2022-12-23)
+
+
+### Features
+
+* add move to drawer for script and flows ([f73dbd8](https://github.com/windmill-labs/windmill/commit/f73dbd8039b3c987ca94e5b56f0ecdea93cbd1b8))
+* add operator mode ([3485b07](https://github.com/windmill-labs/windmill/commit/3485b07b2548b7ea8fbd2b6b31b91e2d36d072ef))
+* auto-invite from same domain ([2bae50f](https://github.com/windmill-labs/windmill/commit/2bae50f3910a99a87efa402a9eef566320fe1f68))
+* **backend:** add SUPERADMIN_SECRET as an env set superadmin ([c283112](https://github.com/windmill-labs/windmill/commit/c28311242d58af12a039b81a5e5c90688022ce8c))
+* **frontend:** Add an input field to edit inline script name ([#1033](https://github.com/windmill-labs/windmill/issues/1033)) ([95a0b9c](https://github.com/windmill-labs/windmill/commit/95a0b9ceae73e291a0def340e935658b6c2ac3a5))
+* **frontend:** Add app number input ([#1010](https://github.com/windmill-labs/windmill/issues/1010)) ([2fe927f](https://github.com/windmill-labs/windmill/commit/2fe927f7fdc1309c7bad8b90fb7e0cc41d364b3f))
+* **frontend:** Add form component + fix connection bug ([#1012](https://github.com/windmill-labs/windmill/issues/1012)) ([424c31c](https://github.com/windmill-labs/windmill/commit/424c31c54a2652b89f9b06499a5aaf1cc0f00ad9))
+* **frontend:** Add select component to app builder ([#1021](https://github.com/windmill-labs/windmill/issues/1021)) ([08071bb](https://github.com/windmill-labs/windmill/commit/08071bb66b4fc40e3b984ffb459e5d52d5816298))
+* **frontend:** Add the ability to lock components so they don't move around ([#1035](https://github.com/windmill-labs/windmill/issues/1035)) ([26a6de2](https://github.com/windmill-labs/windmill/commit/26a6de247c3566bfa524b8fa4f8fc212ca557874))
+* **frontend:** Align output panel UI ([#1025](https://github.com/windmill-labs/windmill/issues/1025)) ([0e871ca](https://github.com/windmill-labs/windmill/commit/0e871ca8432d4f0bc68543b4a3f3bf8f8af99669))
+* **frontend:** App builder password and date input ([#1022](https://github.com/windmill-labs/windmill/issues/1022)) ([4651c9d](https://github.com/windmill-labs/windmill/commit/4651c9d8cd644e59bfd4f57be0bcecc01962a536))
+* **frontend:** AppTable v2 + Inline script panel ([#1023](https://github.com/windmill-labs/windmill/issues/1023)) ([f6df3ae](https://github.com/windmill-labs/windmill/commit/f6df3ae36748a1271625c3f4b50ca66f604d79f7))
+* **frontend:** Fix component synchro ([#1038](https://github.com/windmill-labs/windmill/issues/1038)) ([cebbc5f](https://github.com/windmill-labs/windmill/commit/cebbc5fbd1b8b855c9b1bcab535cff5b9de8d778))
+* **frontend:** Fix inline script status ([#1034](https://github.com/windmill-labs/windmill/issues/1034)) ([be74311](https://github.com/windmill-labs/windmill/commit/be743117d155afb2a2f0fe33ff610e0f621409f7))
+* **frontend:** Fix UI ([#1009](https://github.com/windmill-labs/windmill/issues/1009)) ([0ceb4ab](https://github.com/windmill-labs/windmill/commit/0ceb4ab1a893fecf9e64497612e6040d0e7bc8cd))
+* **frontend:** Fork + Fix table ([#1037](https://github.com/windmill-labs/windmill/issues/1037)) ([ab13e8c](https://github.com/windmill-labs/windmill/commit/ab13e8cce44ded7e05a8dda3d4d4d1ac696bf739))
+* **frontend:** Small UI fixes ([#1026](https://github.com/windmill-labs/windmill/issues/1026)) ([ebca9f3](https://github.com/windmill-labs/windmill/commit/ebca9f39eab27dda65d0ee5de175a90363bfebae))
+* **frontend:** templatable editor with autocompletion ([e228c64](https://github.com/windmill-labs/windmill/commit/e228c6448ead4a7aef433f4abdfe3c466a0f50f4))
+* implement usage tracker + quotas ([fd87109](https://github.com/windmill-labs/windmill/commit/fd871093f0ea4b2def351857d7d8d7e4e79f9539))
+* introduce folders, deprecate items owned by groups ([4329d25](https://github.com/windmill-labs/windmill/commit/4329d259887da71eb2b2a67f73947b0fbe9f3941))
+* introduce folders, deprecate items owned by groups ([c1b0b64](https://github.com/windmill-labs/windmill/commit/c1b0b64e1728007b364d2a0acc58fc459e49e461))
+* Superadmins workspace ([#1003](https://github.com/windmill-labs/windmill/issues/1003)) ([4004de0](https://github.com/windmill-labs/windmill/commit/4004de06180868af4570668a2040bd711a461e0d))
+
+
+### Bug Fixes
+
+* **frontend:** copy-to-clipnoard url with protocol ([#1027](https://github.com/windmill-labs/windmill/issues/1027)) ([f77fe7b](https://github.com/windmill-labs/windmill/commit/f77fe7b6b321c3d00a51a42a4118fd37f7c9d782))
+* **frontend:** Fix AppTable frontend search ([#1013](https://github.com/windmill-labs/windmill/issues/1013)) ([f7627b5](https://github.com/windmill-labs/windmill/commit/f7627b5f17a9f5a4528715eebb4d207f33609da2))
+
+## [1.55.0](https://github.com/windmill-labs/windmill/compare/v1.54.0...v1.55.0) (2022-12-09)
+
+
+### Features
+
+* **frontend:** Add text input to app builder ([#1008](https://github.com/windmill-labs/windmill/issues/1008)) ([6198383](https://github.com/windmill-labs/windmill/commit/6198383138929237c1eb898954a1fd91bdded08a))
+
+## [1.54.0](https://github.com/windmill-labs/windmill/compare/v1.53.0...v1.54.0) (2022-12-08)
+
+
+### Features
+
+* add lockable version to scripts inside flows ([#972](https://github.com/windmill-labs/windmill/issues/972)) ([799fa92](https://github.com/windmill-labs/windmill/commit/799fa925b39316f6f8232d01959c35c4d6fa9533))
+* **frontend:** Add support for object editor + fix wording ([#1004](https://github.com/windmill-labs/windmill/issues/1004)) ([a562dee](https://github.com/windmill-labs/windmill/commit/a562dee3cebfc07f72f0e952cb102c4c86022937))
+* implement flow as a flow step ([8c1c508](https://github.com/windmill-labs/windmill/commit/8c1c5083585f4882aac3f05f71ad1a6414772082))
+
+## [1.53.0](https://github.com/windmill-labs/windmill/compare/v1.52.0...v1.53.0) (2022-12-05)
+
+
+### Features
+
+* add include_header to pass request headers to script ([31c317b](https://github.com/windmill-labs/windmill/commit/31c317b3581e24aa24fa41a708f080c1d1de7e0c))
+* **cli:** hub sync ([#975](https://github.com/windmill-labs/windmill/issues/975)) ([2265372](https://github.com/windmill-labs/windmill/commit/22653727a4106fa604796b3958efab94762041c2))
+* **frontend:** Add app preview ([#993](https://github.com/windmill-labs/windmill/issues/993)) ([c9ad638](https://github.com/windmill-labs/windmill/commit/c9ad63895891ab3bbaeab43a008573f5bd3681b5))
+* **frontend:** clarified UX for connect step ([e4839e2](https://github.com/windmill-labs/windmill/commit/e4839e21ff5d60bec4499245742f2400168c70ad))
+* **frontend:** introduce mysql as a script language ([#982](https://github.com/windmill-labs/windmill/issues/982)) ([e089109](https://github.com/windmill-labs/windmill/commit/e089109b50bd014c7a4f0fd7f60c53e8be63fb95))
+* refactor favorite menu ([c55fae5](https://github.com/windmill-labs/windmill/commit/c55fae54dd043eb1c01a15c8005e29166a4e992b))
+
+
+### Bug Fixes
+
+* **cli:** Fix cli pull push ([#985](https://github.com/windmill-labs/windmill/issues/985)) ([1bac237](https://github.com/windmill-labs/windmill/commit/1bac23785cb6af255732b1a2551bf9ffa00e24e7))
+* **frontend:** Align hub flow list + fix drawer content everywhere ([#991](https://github.com/windmill-labs/windmill/issues/991)) ([9f59a16](https://github.com/windmill-labs/windmill/commit/9f59a160c39048447ffeefc5070c52e8692c8316))
+* **frontend:** Fix app InputValue sync ([#994](https://github.com/windmill-labs/windmill/issues/994)) ([e217fbf](https://github.com/windmill-labs/windmill/commit/e217fbf071fa834c4b4288f602125164bf1d93bf))
+* **frontend:** fix app preview ([#979](https://github.com/windmill-labs/windmill/issues/979)) ([129a0ad](https://github.com/windmill-labs/windmill/commit/129a0ad56b58840620fdc77e619928e04c67cd1f))
+* **frontend:** fix home ([#981](https://github.com/windmill-labs/windmill/issues/981)) ([fa64e83](https://github.com/windmill-labs/windmill/commit/fa64e83f7ea6bc7786a15db647319d2f2a322b5b))
+* **frontend:** fix home header ([#977](https://github.com/windmill-labs/windmill/issues/977)) ([e9fa0ad](https://github.com/windmill-labs/windmill/commit/e9fa0ad0b75d0678167e7a48f8406639e85986a9))
+* **frontend:** Fix home margins ([#992](https://github.com/windmill-labs/windmill/issues/992)) ([62d2a33](https://github.com/windmill-labs/windmill/commit/62d2a3343dc27317f33446918404373b7d8285f5))
+* **frontend:** Make context clickable ([#984](https://github.com/windmill-labs/windmill/issues/984)) ([9264f4b](https://github.com/windmill-labs/windmill/commit/9264f4b233858537bb344355c5be43be3ec9d8d9))
+* **frontend:** variables and resources uses tab navigation ([90ce431](https://github.com/windmill-labs/windmill/commit/90ce4314181d8e5031c08d5fbb75b920c33b7f75))
+
+## [1.52.0](https://github.com/windmill-labs/windmill/compare/v1.51.0...v1.52.0) (2022-12-02)
+
+
+### Features
+
+* add favorite/star + remove flows/scripts page in favor of unified home page ([#968](https://github.com/windmill-labs/windmill/issues/968)) ([f3f694e](https://github.com/windmill-labs/windmill/commit/f3f694e9251fc62d8e3e10497e8936c588b456ba))
+* **cli:** improved setup & allow workspace in base url & refactor workspaces/remotes to unify ([#966](https://github.com/windmill-labs/windmill/issues/966)) ([d3a171c](https://github.com/windmill-labs/windmill/commit/d3a171c28355c5d452e6e9caa0aa741c1ff23875))
+* **cli:** Login via Frontend ([#956](https://github.com/windmill-labs/windmill/issues/956)) ([2c31a9c](https://github.com/windmill-labs/windmill/commit/2c31a9cbdf84ff2659313df799cbd79f9c167325))
+* **deno-client:** support mysql ([#971](https://github.com/windmill-labs/windmill/issues/971)) ([0e402f6](https://github.com/windmill-labs/windmill/commit/0e402f6a9dfd1b6d00f6d2a951740d7aea0a8b70))
+* **frontend:** Add actions to tables ([#951](https://github.com/windmill-labs/windmill/issues/951)) ([1069105](https://github.com/windmill-labs/windmill/commit/10691054510dd955a6f0d36c0186fdab9ce0facc))
+* **frontend:** Add Mailchimp resource instructions ([#967](https://github.com/windmill-labs/windmill/issues/967)) ([ba90e8c](https://github.com/windmill-labs/windmill/commit/ba90e8c1b8131e1b1e38322d165c04a53a8622b2))
+* **frontend:** flow status viewer include a graph ([02a9c5c](https://github.com/windmill-labs/windmill/commit/02a9c5c4eac557486df6908536a8467d68b92eca))
+* **frontend:** rework script detail ([#952](https://github.com/windmill-labs/windmill/issues/952)) ([6c45fe7](https://github.com/windmill-labs/windmill/commit/6c45fe7344858761422916cc497018b35753e0ce))
+* **frontend:** Update app component list ([#947](https://github.com/windmill-labs/windmill/issues/947)) ([ec1cebc](https://github.com/windmill-labs/windmill/commit/ec1cebc7920350939e365322f77898b31cafd795))
+* overhaul scripts and flows page ([4946093](https://github.com/windmill-labs/windmill/commit/494609364c9d6109c08c7531cf02223793325f88))
+* overhaul scripts and flows page ([c26be86](https://github.com/windmill-labs/windmill/commit/c26be86cef9d6cad44ae7cbbb5e0fd5d147c5c52))
+* **python:** add support for parsing resource type in python ([63d95cf](https://github.com/windmill-labs/windmill/commit/63d95cfbb31a2b599fa9deaee203e1c4c2f0715e))
+* refactor variable + resource linkage + OAuth visibility ([37967a7](https://github.com/windmill-labs/windmill/commit/37967a795006c2eb4e8b218abb3d1b0525c17d5e))
+* unify resources under a single connect API ([539d6be](https://github.com/windmill-labs/windmill/commit/539d6be9088ccb2d18b0d16ca020b23bffaa79b9))
+
+
+### Bug Fixes
+
+* **backend:** support PIP_INDEX_URL ([12f9677](https://github.com/windmill-labs/windmill/commit/12f967726b96cc04e5024134216727ddfcd5fe82))
+* **backend:** support PIP_INDEX_URL ([afcb44a](https://github.com/windmill-labs/windmill/commit/afcb44a12707dc3b0839182479438d2b010362ca))
+* **frontend:** Fix pie animation + actions wrap ([#953](https://github.com/windmill-labs/windmill/issues/953)) ([ed7838d](https://github.com/windmill-labs/windmill/commit/ed7838d6bcf538525f6b3e4257bffe6d51318c8a))
+* **frontend:** psql demo expects integers as a key ([#958](https://github.com/windmill-labs/windmill/issues/958)) ([4d8a5c4](https://github.com/windmill-labs/windmill/commit/4d8a5c4fd927e421825a9d9d2dc5dcfaf8b3949a))
+* **frontend:** Refactor apps to support multiple breakpoints ([#957](https://github.com/windmill-labs/windmill/issues/957)) ([96666af](https://github.com/windmill-labs/windmill/commit/96666af3d9d6f68e4e5bb0f7a748614c9916f394))
+
+## [1.51.0](https://github.com/windmill-labs/windmill/compare/v1.50.0...v1.51.0) (2022-11-26)
+
+
+### Features
+
+* Add notification on app save ([#943](https://github.com/windmill-labs/windmill/issues/943)) ([79cec36](https://github.com/windmill-labs/windmill/commit/79cec368ba643a88a554a88e4bc0500701e2fcc8))
+* **backend:** add configurable custom client ([975a1db](https://github.com/windmill-labs/windmill/commit/975a1db10ea592038cef0c2677e66a8b6d6b8ee5))
+* **cli:** Run flows & scripts ([#940](https://github.com/windmill-labs/windmill/issues/940)) ([cdd3e2c](https://github.com/windmill-labs/windmill/commit/cdd3e2cfc11cd003246643528b950cd0aafe1140))
+* **frontend:** Add guard against script overwrite ([#944](https://github.com/windmill-labs/windmill/issues/944)) ([dd75b37](https://github.com/windmill-labs/windmill/commit/dd75b370afd3d7e6a112e0ec9a6444a82b5620e3))
+* **frontend:** Add inline script picker to apps ([#945](https://github.com/windmill-labs/windmill/issues/945)) ([ddab2df](https://github.com/windmill-labs/windmill/commit/ddab2dffd5459a3e35a368e09a64ebcbceefc87a))
+* **frontend:** flow UX overhaul II + go + python support for trigger scripts ([#928](https://github.com/windmill-labs/windmill/issues/928)) ([802abe7](https://github.com/windmill-labs/windmill/commit/802abe7f901fc93bee1be401a3166fa22b63d00c))
+* **frontend:** login page makeup ([5028d86](https://github.com/windmill-labs/windmill/commit/5028d8603d08f13f4c9ae061b5aa9c6b4b5ea4f4))
+* **frontend:** login page makeup ([ced2678](https://github.com/windmill-labs/windmill/commit/ced2678a21e2078973cfbe506586061f806c2dfe))
+* Update apps button component with colors ([#936](https://github.com/windmill-labs/windmill/issues/936)) ([4b2b346](https://github.com/windmill-labs/windmill/commit/4b2b3467d2bbb204acd5330c4c100d63acb4e40a))
+
+
+### Bug Fixes
+
+* **backend:** bash flow lock & add flow lock tests ([#933](https://github.com/windmill-labs/windmill/issues/933)) ([4ddb3ec](https://github.com/windmill-labs/windmill/commit/4ddb3ec276ef9140e15a8604d796c3a2e6210311))
+* **deno-client:** pg 0.16.1 -> 0.17.0 ([ac6454b](https://github.com/windmill-labs/windmill/commit/ac6454b3835562f70694ce2b935e4b229f9118c6))
+* **frontend:** add checkbox component + fix alignment ([#941](https://github.com/windmill-labs/windmill/issues/941)) ([43a1d7e](https://github.com/windmill-labs/windmill/commit/43a1d7ef2a1c9167262ea7d19cc0fb10d0493eed))
+* **frontend:** Cleanup dead code ([#935](https://github.com/windmill-labs/windmill/issues/935)) ([fa4840a](https://github.com/windmill-labs/windmill/commit/fa4840ad656b2cb592c644193f617b49e53211aa))
+* **frontend:** Fix context panel + delete component ([#937](https://github.com/windmill-labs/windmill/issues/937)) ([ab481b3](https://github.com/windmill-labs/windmill/commit/ab481b3096ae6390e0d08b23a6b18f0f988cf1bd))
+* **frontend:** prevent runnable to run if the script is not defined ([#938](https://github.com/windmill-labs/windmill/issues/938)) ([e64195e](https://github.com/windmill-labs/windmill/commit/e64195e42b940e552d9b89b040dff4a4d0f8be37))
+* **frontend:** properly refresh context panel + Adjust style in the flow editor ([#934](https://github.com/windmill-labs/windmill/issues/934)) ([b59a1de](https://github.com/windmill-labs/windmill/commit/b59a1de93baade3ad576300c07143fbd3f074054))
+
+## [1.50.0](https://github.com/windmill-labs/windmill/compare/v1.49.1...v1.50.0) (2022-11-21)
+
+
+### Features
+
+* **deno,python:** get/set_shared_state ([c8266fb](https://github.com/windmill-labs/windmill/commit/c8266fb8b3262d9e9ec5698f824b2e9df716a228))
+* **frontend:** overhaul the whole flow UX ([d23e218](https://github.com/windmill-labs/windmill/commit/d23e218e1fd9b200aaa3fff12182f18e251da796))
+
+
+### Bug Fixes
+
+* **caching:** preserve permissions ([a352975](https://github.com/windmill-labs/windmill/commit/a3529759ad34db5c8234a7886aba1c3d07a644cf))
+
+## [1.49.1](https://github.com/windmill-labs/windmill/compare/v1.49.0...v1.49.1) (2022-11-20)
+
+
+### Bug Fixes
+
+* **caching:** add a second caching mechanism by tarring the entire cache for fast startup ([7af345e](https://github.com/windmill-labs/windmill/commit/7af345e5e57c6fbc35db9069782432664232851a))
+
+## [1.49.0](https://github.com/windmill-labs/windmill/compare/v1.48.2...v1.49.0) (2022-11-20)
+
+
+### Features
+
+* **go:** improve cold start of 200ms by building outside of nsjail ([838a92a](https://github.com/windmill-labs/windmill/commit/838a92a0dbb75f4e7e32a7541800cbda4808cea7))
+* **python-client:** remove unecessary imports in wmill to speed-up imports ([46fe9ad](https://github.com/windmill-labs/windmill/commit/46fe9ad52594d3a45b7917b91b37a83bc779bb1b))
+
+## [1.48.2](https://github.com/windmill-labs/windmill/compare/v1.48.1...v1.48.2) (2022-11-19)
+
+
+### Bug Fixes
+
+* **go-client:** support setVariable, setResource, setState, getState ([e33bd1e](https://github.com/windmill-labs/windmill/commit/e33bd1e6b25bb9e3a3fe6f2c93d8c686c200b253))
+
+## [1.48.1](https://github.com/windmill-labs/windmill/compare/v1.48.0...v1.48.1) (2022-11-19)
+
+
+### Bug Fixes
+
+* **python-client:** get_state on empty state return None ([968675d](https://github.com/windmill-labs/windmill/commit/968675d8d068b19413a8bca7d4cb80179646c114))
+
+## [1.48.0](https://github.com/windmill-labs/windmill/compare/v1.47.3...v1.48.0) (2022-11-18)
+
+
+### Features
+
+* add slack_bot token on connecting workspace to slack ([b3178d1](https://github.com/windmill-labs/windmill/commit/b3178d1b8aacfa90b8a68554a186f3b26f3190ba))
+* **backend:** sync cache features on all workers [enterprise] ([#907](https://github.com/windmill-labs/windmill/issues/907)) ([bd09884](https://github.com/windmill-labs/windmill/commit/bd09884955bbe04f41fbcce9b978a070145f23a3))
+* **python:** add Resource[resource_type] as a parsed parameter ([9d17abb](https://github.com/windmill-labs/windmill/commit/9d17abbb12463c81de325eef875161cf86449b25))
+* supercache extended to all version ([8846ca5](https://github.com/windmill-labs/windmill/commit/8846ca585699c2ec7b18b4479e895b296774ee95))
+
+
+### Bug Fixes
+
+* **backend:** saving bash script does not require dep job ([381b036](https://github.com/windmill-labs/windmill/commit/381b0368d72ad42501082c91a7c62964593ba3ad))
+* **frontend:** app editor v1 ([#908](https://github.com/windmill-labs/windmill/issues/908)) ([53a8c5e](https://github.com/windmill-labs/windmill/commit/53a8c5e04cc4f407c137b0d621003dbab1bfdc67))
+* **frontend:** Reduce the size of the separator + fix Auto scroll ([#895](https://github.com/windmill-labs/windmill/issues/895)) ([3f8295b](https://github.com/windmill-labs/windmill/commit/3f8295bb0c7d9e9c831e8dbcb7f1e8b944e45c66))
+* support flows to be triggered by slack commands ([199a11a](https://github.com/windmill-labs/windmill/commit/199a11a8cf92691a3ac5aa7ebdc3157d10677139))
+
+## [1.47.3](https://github.com/windmill-labs/windmill/compare/v1.47.2...v1.47.3) (2022-11-15)
+
+
+### Bug Fixes
+
+* **python-client:** fix transform_leaves ([a649f77](https://github.com/windmill-labs/windmill/commit/a649f772a564eaffb5f6192a510f7112ed618300))
+
+## [1.47.2](https://github.com/windmill-labs/windmill/compare/v1.47.1...v1.47.2) (2022-11-15)
+
+
+### Bug Fixes
+
+* **python-client:** fix get_state ([b4fd470](https://github.com/windmill-labs/windmill/commit/b4fd4700251892116b0dff2940d98b7e473c79bf))
+
+## [1.47.1](https://github.com/windmill-labs/windmill/compare/v1.47.0...v1.47.1) (2022-11-15)
+
+
+### Bug Fixes
+
+* **python-client:** fix set_resource ([a6a5ada](https://github.com/windmill-labs/windmill/commit/a6a5adadf45f6334eaf17f59985c0e7870f25167))
+
+## [1.47.0](https://github.com/windmill-labs/windmill/compare/v1.46.2...v1.47.0) (2022-11-15)
+
+
+### Features
+
+* **backend:** Flow lock ([#868](https://github.com/windmill-labs/windmill/issues/868)) ([47c9ff1](https://github.com/windmill-labs/windmill/commit/47c9ff1edc28b63a1a16ffce08d3751a4f8f5422))
+* **backend:** remove go.sum from go lockfile ([#891](https://github.com/windmill-labs/windmill/issues/891)) ([3357cff](https://github.com/windmill-labs/windmill/commit/3357cffb043254d8712a2afe2729533d5884d56f))
+* **clients:** rename internal state as state + setters for resources/variables in python ([32bca1f](https://github.com/windmill-labs/windmill/commit/32bca1fd4cd0714a9f18a508b0e0782f63ee25a8))
+
+
+### Bug Fixes
+
+* **backend:** go use windmill cache dir even if nsjail disabled ([a9abd28](https://github.com/windmill-labs/windmill/commit/a9abd288822731add05d00e3d3fc43d29e11c7cb))
+* **frontend:** add size prop to tabs ([#894](https://github.com/windmill-labs/windmill/issues/894)) ([e8d3a0e](https://github.com/windmill-labs/windmill/commit/e8d3a0efb1e23ae66d755489f96f09932544be9c))
+* **frontend:** App Editor v0 ([#886](https://github.com/windmill-labs/windmill/issues/886)) ([cc5f629](https://github.com/windmill-labs/windmill/commit/cc5f629a7b142a2bd0ce7ca8950e24f6cb5473ff))
+* **frontend:** Set settings as header and error handler as footer ([#893](https://github.com/windmill-labs/windmill/issues/893)) ([4dc05b9](https://github.com/windmill-labs/windmill/commit/4dc05b913e4d98dd37b032639831d20aa662e4e9))
+
+## [1.46.2](https://github.com/windmill-labs/windmill/compare/v1.46.1...v1.46.2) (2022-11-12)
+
+
+### Bug Fixes
+
+* **ci:** sqlx offline data ([76a6768](https://github.com/windmill-labs/windmill/commit/76a6768ed9ab223363f47c62cfcd8c51dd624b62))
+
+## [1.46.1](https://github.com/windmill-labs/windmill/compare/v1.46.0...v1.46.1) (2022-11-12)
+
+
+### Bug Fixes
+
+* **backend:** apps backend v0 ([#888](https://github.com/windmill-labs/windmill/issues/888)) ([2d9e990](https://github.com/windmill-labs/windmill/commit/2d9e9909da5b82eda39eb99c870f073b869b6ff5))
+
+## [1.46.0](https://github.com/windmill-labs/windmill/compare/v1.45.0...v1.46.0) (2022-11-12)
+
+
+### Features
+
+* **cli:** Relax push folder layout to accept one layer of organizational structure ([#882](https://github.com/windmill-labs/windmill/issues/882)) ([a658308](https://github.com/windmill-labs/windmill/commit/a658308b59d7ef51d1aa6cda7598947ed0ce7548))
+* **cli:** Tarball pull ([#867](https://github.com/windmill-labs/windmill/issues/867)) ([d375836](https://github.com/windmill-labs/windmill/commit/d375836989fd730acbb4a04218d143b9fef63e0d))
+* deprecate previous_result in favor of results per id ([40183ce](https://github.com/windmill-labs/windmill/commit/40183ce4e42f648d9eb6e2765fb141e16eba908e))
+* **frontend:** Flow graph ([#827](https://github.com/windmill-labs/windmill/issues/827)) ([9bf0f6e](https://github.com/windmill-labs/windmill/commit/9bf0f6e70d7501737a61e4d62d116d44b1f136df))
+* publish arm64 image ([#885](https://github.com/windmill-labs/windmill/issues/885)) ([c3b2bab](https://github.com/windmill-labs/windmill/commit/c3b2bab5d1a7eee49c517c2c8c5e9108c3f32333))
+
 ## [1.45.0](https://github.com/windmill-labs/windmill/compare/v1.44.0...v1.45.0) (2022-11-06)
 
 

Dockerfile

@@ -1,8 +1,10 @@
-FROM python:3.11.0-slim-buster as nsjail
+FROM debian:buster-slim as nsjail
 
 WORKDIR /nsjail
 
-RUN apt-get -y update \
+ARG nsjail=""
+
+RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
     bison=2:3.3.* \
     flex=2.6.* \
@@ -13,11 +15,12 @@ RUN apt-get -y update \
     libnl-route-3-dev=3.4.* \
     make=4.2.* \
     pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*
+    protobuf-compiler=3.6.*; fi
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
-    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
-RUN make
+
+RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
+RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
 FROM rust:slim-buster AS rust_base
 
@@ -29,7 +32,7 @@ RUN apt-get -y update \
 
 RUN rustup component add rustfmt
 
-RUN cargo install cargo-chef 
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef 
 
 WORKDIR /windmill
 
@@ -48,60 +51,77 @@ COPY frontend .
 RUN mkdir /backend
 COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
 COPY /openflow.openapi.yaml /openflow.openapi.yaml
+COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
+RUN cd /backend/windmill-api && . ./build_openapi.sh
+
 RUN npm run generate-backend-client
 ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
 RUN npm run check
 
 
+
 FROM rust_base AS planner
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
 
-RUN cargo chef prepare  --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
 
 FROM rust_base AS builder
+ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
 
 COPY --from=frontend /frontend /frontend
+COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml
 COPY .git/ .git/
 
-RUN cargo build --release
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:buster-slim
+FROM python:3.11.1-slim-buster
+
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
-    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
-    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
-    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    curl -o rclone.zip "https://downloads.rclone.org/v1.60.1/rclone-v1.60.1-linux-$arch.zip"; \
+    unzip -p rclone.zip rclone-v1.60.1-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
+    chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone
+
+RUN set -eux; \
+    arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    url=; \
+    case "$arch" in \
+    'amd64') \
+    targz='go1.19.3.linux-amd64.tar.gz'; \
+    ;; \
+    'arm64') \
+    targz='go1.19.3.linux-arm64.tar.gz'; \
+    ;; \
+    'armhf') \
+    targz='go1.19.3.linux-armv6l.tar.gz'; \
+    ;; \
+    *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
+    esac; \
+    wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz";
+
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
 ENV TZ=Etc/UTC
 
-ENV PYTHON_VERSION 3.10.4
-
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
-    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
-    && make -j 4 && make install
-
 RUN /usr/local/bin/python3 -m pip install pip-tools
-RUN /usr/local/bin/python3 -m pip install nltk
-RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 

README.md

@@ -5,7 +5,7 @@
     <em>.</em>
 </p>
 <p align=center>
-Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger flows and scripts as internal apps. Convert code to no-code modules and, if the auto-generated UI is not sufficient, use it solely as an highly scalable backend layer. Add automation to your product or build your own no-code tool and delegate the core layer to Windmill. The script languages supported are: Python, Typescript and Go.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash.
 </p>
 
 <p align="center">
@@ -22,8 +22,7 @@ Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks an
 
 ---
 
-**Join the beta (personal workspaces are free forever)**:
-<https://app.windmill.dev>
+**Try it (personal workspaces are free forever)**: <https://app.windmill.dev>
 
 **Documentation**: <https://docs.windmill.dev>
 
@@ -35,15 +34,10 @@ Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks an
 
 **Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
 
-**[Self-host instruction](#how-to-self-host)**
-
 You can show your support for the project by starring this repo.
 
----
-
-Windmill Labs offers commercial licenses and support to convert your existing
-automation and help you scale it in production. If interested, contact
-ruben@windmill.dev (founder of Windmill).
+Windmill Labs offers commercial licenses, an enterprise edition, local hub
+mirrors, and support: contact ruben@windmill.dev.
 
 ---
 
@@ -53,59 +47,72 @@ ruben@windmill.dev (founder of Windmill).
 <b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
 </p>
 
+![Windmill Screenshot](./imgs/windmill-flow.png)
 ![Windmill Screenshot](./imgs/windmill.png)
 
-Windmill is <b>fully open-sourced</b>:
+Windmill is <b>fully open-sourced (AGPLv3)</b>:
 
-- `community/`, `python-client/` and `deno-client/` are Apache 2.0
-- backend, frontend and everything else under AGPLv3.
+- [Windmill](#windmill)
+  - [Main Concepts](#main-concepts)
+  - [CLI](#cli)
+  - [Layout](#layout)
+  - [Stack](#stack)
+  - [Security](#security)
+    - [Sandboxing and workload isolation](#sandboxing-and-workload-isolation)
+    - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
+  - [Performance](#performance)
+  - [Architecture](#architecture)
+    - [Big-picture Architecture](#big-picture-architecture)
+    - [Technical Architecture](#technical-architecture)
+  - [How to self-host](#how-to-self-host)
+    - [Docker compose](#docker-compose)
+    - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
+    - [Commercial license](#commercial-license)
+    - [OAuth for self-hosting (very optional)](#oauth-for-self-hosting-very-optional)
+    - [Resource types](#resource-types)
+  - [Environment Variables](#environment-variables)
+  - [Run a local dev setup](#run-a-local-dev-setup)
+    - [only Frontend](#only-frontend)
+    - [Backend + Frontend](#backend--frontend)
+  - [Contributors](#contributors)
+  - [Copyright](#copyright)
 
-## What is the general idea behind Windmill
+## Main Concepts
 
-1. Define a minimal and generic script in Python, Typescript or Go that solves a
-   specific task. Here sending an email with SMTP. The code can be defined in
-   the provided Web IDE or synchronized with your own github repo:
-   ![Step 1](./imgs/python-script.png)
+1. Define a minimal and generic script in Python, Typescript, Go or Bash that
+   solves a specific task. Here sending an email with SMTP. The code can be
+   defined in the provided Web IDE or synchronized with your own github repo:
+   ![Step 1](./imgs/windmill-editor.png)
 
-2. Your scripts parameters are automatically parsed and generate a frontend. You
-   can narrow down the types during task definition to specify regex for string,
-   an enum or a specific format for objects. Each script correspond to an app by
-   itself: ![Step 2](./imgs/arguments.png)
+2. Your scripts parameters are automatically parsed and generate a frontend.
+   ![Step 2](./imgs/windmill-run.png) ![Step 3](./imgs/windmill-result.png)
 
 3. Make it flow! You can chain your scripts or scripts made by the community
-   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
-   between Windmill and the hub to make it easy to build flows from a soon-to-be
-   exhaustive library of generic modules. In flows, one can pipe output to input
-   using "Dynamic" expressions that are just plain Javascript underneath. Flows
-   can contain for-loops, branching (coming soon). As such and coupled with
-   inputs being able to refer to any step's output, they are actual DAG rather
-   than just linear sequences. They are backed by an open JSON spec we call
-   [OpenFlow](https://docs.windmill.dev/docs/openflow)
+   shared on [WindmillHub](https://hub.windmill.dev).
+   ![Step 4](./imgs/windmill-flow.png)
 
-Both scripts and flows are not restricted to be triggered by the UI. They can be
-triggered by a schedule, watch for changes (using
-[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
-triggered through API with either an async or sync webhook. The latter kind of
-endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
-central place to host, build and run all of your integrations, automation and
-internal apps. We include credentials management and OAuth integration, groups
-and much more!
+4. Build complex UI on top of your scripts and flows.
+   ![Step 5](./imgs/windmill-builder.png)
+
+Scripts and flows can also be triggered by a cron schedule '*/5 * * * *' or
+through webhooks.
+
+You can build your entire infra on top of Windmill!
+
+## CLI
+
+We have a powerful CLI to interact with the windmill platform and sync your
+scripts from your own github repo. See
+[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
+
+![CLI Screencast](./cli/vhs/output/setup.gif)
 
 ## Layout
 
-- `backend/`: The whole Rust backend
-- `frontend`: The whole Svelte frontend
-- `community/`: Scripts and resource types included in every workspace. It is
-  useful for Python scripts since the [WindmillHub](https://hub.windmill.dev)
-  only allow deno scripts and for sharing resource types that will be included
-  in every workspace.
-- `lsp/`: The lsp asssistant for the monaco editor
-- `nsjail/`: The nsjail configuration files for sandboxing of the scripts'
-  execution
-- `python-client/`: The wmill python client used within scripts to interact with
-  the windmill platform
-- `deno-client/`: The wmill deno client used within scripts to interact with the
-  windmill platform
+- `backend/`: Rust backend
+- `frontend`: Svelte frontend
+- `lsp/`: Lsp asssistant for the monaco editor
+- `<lang>-client/`: Windmill client for the given `<lang>`
 
 ## Stack
 
@@ -124,6 +131,7 @@ and much more!
   the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
+- golang runtime is 1.19.1
 
 ## Security
 
@@ -143,9 +151,11 @@ That is what we do at <https://app.windmill.dev>.
 
 ## Performance
 
-The performances are great, as long as you do not exceed the parrallelism of the
-workers, we are
-[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
+Once a job started, there is no overhead compared to running the same script on
+the node with its corresponding runner (Deno/Go/Python/Bash). The added latency
+from a job being pulled from the queue, started, and then having its result sent
+back to the database is ~50ms. A typical lightweight deno job will take around
+100ms total.
 
 ## Architecture
 
@@ -172,6 +182,8 @@ compiling from source or using without a postgres super user, see
 `docker compose up` with the following docker-compose is sufficient:
 <https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
 
+Go to https://localhost et voilà :)
+
 For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
 nsjail will not be able to launch the isolated scripts.
 
@@ -181,6 +193,10 @@ The default super-admin user is: admin@windmill.dev / changeme
 
 From there, you can create other users (do not forget to change the password!)
 
+### Kubernetes (k8s) and Helm charts
+
+We publish helm charts at: <https://github.com/windmill-labs/windmill-helm-charts>
+
 ### Commercial license
 
 To self-host Windmill, you must respect the terms of the AGPLv3 license which
@@ -195,8 +211,8 @@ be AGPLv3 or you must get a commercial license. Contact us at
 
 In addition, a commercial license grants you a dedicated engineer to transition
 your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permissioning features
-such as groups and the ability to create more than one workspace.
+export features, SSO, unlimited users creation, advanced permission managing
+features such as groups and the ability to create more than one workspace.
 
 ### OAuth for self-hosting (very optional)
 
@@ -215,12 +231,40 @@ the following format:
 
 and mount it at `/usr/src/app/oauth.json`.
 
+The redirect url for the oauth clients is: `<instance_url>/user/login_callback/<client>`
+
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
 [Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
 welcome contributions!
 
+You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
+
+```json
+{
+  "<client>": {
+    "id": "<CLIENT_ID>",
+    "secret": "<CLIENT_SECRET>",
+    // To add a new OAuth2 IdP
+    "login_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "userinfo_url": "<userinfo endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    },
+    // To add a new OAuth2 Resource
+    "connect_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    }
+  }
+}
+```
+
 ### Resource types
 
 You will also want to import all the approved resource types from
@@ -228,6 +272,47 @@ You will also want to import all the approved resource types from
 automatically currently, but it will be possible using a command with the
 upcoming CLI tool.
 
+## Environment Variables
+
+| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          |                       |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker)                                                                                                                | Worker                |
+| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                   | 300                    | The timeout in seconds for the execution of a script                                                                                                                                               | Worker                |
+| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| SERVE_CSP                 | None                   | The CSP directives to use when serving the frontend static assets                                                                                                                                  | Server                |
+| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+|                           |
+
+
+
+
 ## Run a local dev setup
 
 ### only Frontend
@@ -247,9 +332,9 @@ See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
 running options.
 
 1. Create a Postgres Database for Windmill and create an admin role inside your
-   Postgres setup.
-   The easiest way to get a working postgres is running `cargo install sqlx-cli && sqlx migrate run`.
-   This will also avoid compile time issue with sqlx's `query!` macro
+   Postgres setup. The easiest way to get a working postgres is running
+   `cargo install sqlx-cli && sqlx migrate run`. This will also avoid compile
+   time issue with sqlx's `query!` macro
 2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
    your PATH
 3. Install deno and python3, have the bins at `/usr/bin/deno` and
@@ -258,7 +343,8 @@ running options.
 5. Install the [lld linker](https://lld.llvm.org/)
 6. Go to `frontend/`:
    1. `npm install`, `npm run generate-backend-client` then `npm run dev`
-   2. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will crash
+   2. In another shell `npm run build` otherwise the backend will not find the
+      `frontend/build` folder and will crash
    3. In another shell `sudo caddy run --config Caddyfile`
 7. Go to `backend/`:
    `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`

backend/.gitignore

@@ -1,3 +1,4 @@
 target/
 .env
 oauth.json
+windmill-api/openapi-deref.yaml

backend/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.45.0"
+version = "1.60.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.45.0"
+version = "1.60.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -27,6 +27,9 @@ edition = "2021"
 name = "windmill"
 path = "./src/main.rs"
 
+[features]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+
 [dependencies]
 anyhow.workspace = true
 tokio.workspace = true
@@ -41,6 +44,10 @@ sqlx.workspace = true
 tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
+git-version.workspace = true
+rsa.workspace = true
+base64.workspace = true
+sha2.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -134,3 +141,7 @@ futures = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
+const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
+dyn-iter = "0.2.0"
+rsa = "0.7.2"
+async-stripe = { version = "0.14", features = ["runtime-tokio-hyper", "checkout"] }

backend/build.rs

@@ -0,0 +1,5 @@
+// generated by `sqlx migrate build-script`
+fn main() {
+    // trigger recompilation when a new migration is added
+    println!("cargo:rerun-if-changed=migrations");
+}

backend/migrations/20221024225533_apps.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE app;
+DROP TABLE app_version;
+DROP TYPE EXECUTION_MODE;

backend/migrations/20221024225533_apps.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+CREATE TABLE app (
+    id BIGSERIAL PRIMARY KEY,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+	path varchar(255) NOT NULL,
+    summary VARCHAR(1000) NOT NULL DEFAULT '',
+    policy JSONB NOT NULL,
+    versions BIGINT[] NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE app_version(
+    id BIGSERIAL PRIMARY KEY,
+    flow_id BIGINT NOT NULL,
+    value JSONB NOT NULL,
+	created_by VARCHAR(50) NOT NULL,
+	created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    FOREIGN KEY (flow_id) REFERENCES app(id) ON DELETE CASCADE
+);
+
+CREATE POLICY see_own ON app FOR ALL
+USING (SPLIT_PART(app.path, '/', 1) = 'u' AND SPLIT_PART(app.path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON app FOR ALL
+USING (SPLIT_PART(app.path, '/', 1) = 'g' AND SPLIT_PART(app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20221104105107_queue-raw_lock.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE queue DROP raw_lock;

backend/migrations/20221104105107_queue-raw_lock.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN raw_lock TEXT DEFAULT NULL;

backend/migrations/20221104111317_completed_job-raw_lock.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE queue DROP raw_lock;

backend/migrations/20221104111317_completed_job-raw_lock.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE completed_job ADD COLUMN raw_lock TEXT DEFAULT NULL;

backend/migrations/20221104135246_flow-dependency_job.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE flow DROP dependency_job;

backend/migrations/20221104135246_flow-dependency_job.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE flow ADD COLUMN dependency_job UUID DEFAULT NULL;
+ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'flowdependencies';

backend/migrations/20221123151919_grant_sequence_privileges.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221123151919_grant_sequence_privileges.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_user;
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_admin;

backend/migrations/20221126181131_pg_rt_name.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221126181131_pg_rt_name.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+UPDATE resource_type rt SET name = 'postgresql' WHERE name = 'postgres' AND  NOT EXISTS (
+   SELECT 1 FROM resource_type WHERE name = 'postgresql' AND rt.workspace_id = workspace_id
+);
+
+UPDATE resource SET resource_type = 'postgresql' WHERE resource_type = 'postgres';

backend/migrations/20221127093847_add_account_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127093847_add_account_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE account ADD COLUMN refresh_error TEXT;

backend/migrations/20221127143814_remove_oauth_from_res.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127143814_remove_oauth_from_res.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE resource DROP COLUMN is_oauth;

backend/migrations/20221128210151_favorites.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221128210151_favorites.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+
+CREATE TYPE FAVORITE_KIND AS ENUM ('app', 'script', 'flow');
+
+CREATE TABLE favorite (
+    usr VARCHAR(50) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    path VARCHAR(255) NOT NULL,
+    favorite_kind FAVORITE_KIND NOT NULL,
+    PRIMARY KEY (usr, workspace_id, favorite_kind, path)
+);

backend/migrations/20221203080006_apps_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221203080006_apps_2.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE app ADD CONSTRAINT unique_path_workspace_id UNIQUE (workspace_id, path);
+ALTER TABLE app ENABLE ROW LEVEL SECURITY;

backend/migrations/20221207103910_admins_workspace.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207103910_admins_workspace.up.sql

@@ -0,0 +1,9 @@
+INSERT INTO workspace(id, name, owner) VALUES
+	('admins', 'Admins', 'admin@windmill.dev');
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('admins');
+
+INSERT INTO workspace_key
+	(workspace_id, kind, key)
+	VALUES ('admins', 'cloud', md5(random()::text) || md5(random()::text))

backend/migrations/20221207125219_admins_hub_sync.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207125219_admins_hub_sync.up.sql

@@ -0,0 +1,20 @@
+INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'admins',
+'system',
+'import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
+
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'Syncronize Hub Resource types with starter workspace',
+'Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.',
+'u/admin/hub_sync', -28028598712388162, 'deno', '');

backend/migrations/20221208135921_job_pre_run_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221208135921_job_pre_run_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN pre_run_error text;

backend/migrations/20221210125521_increase_variable_length.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210125521_increase_variable_length.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000);

backend/migrations/20221210194626_usage_tracker.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210194626_usage_tracker.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE usage (
+    id VARCHAR(50) NOT NULL,
+    is_workspace BOOLEAN NOT NULL,
+    month_ INTEGER NOT NULL,
+    usage INTEGER NOT NULL,
+    PRIMARY KEY (id, is_workspace, month_)
+);

backend/migrations/20221211055346_emails_everywhere.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN email; 
+ALTER TABLE workspace_settings DROP COLUMN slack_email; 
+ALTER TABLE schedule DROP COLUMN email; 
+ALTER TABLE schedule DROP COLUMN error; 

backend/migrations/20221211055346_emails_everywhere.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 
+ALTER TABLE workspace_settings ADD COLUMN slack_email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';;
+ALTER TABLE schedule ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';  
+ALTER TABLE schedule ADD COLUMN error TEXT;  
+

backend/migrations/20221211192539_grant_usage.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_domain;

backend/migrations/20221211192539_grant_usage.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_admin;
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_user;

backend/migrations/20221211200923_remove_admin_from_starters.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221211200923_remove_admin_from_starters.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+DELETE FROM usr WHERE workspace_id = 'starter' AND email = 'admin@windmill.dev';

backend/migrations/20221212114409_add_all_group_to_admins.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212114409_add_all_group_to_admins.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO group_ SELECT id, 'all', 'The group that always contains all users of this workspace' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20221212130731_add_autoinvite.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212130731_add_autoinvite.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_domain VARCHAR(50);
+ALTER TABLE workspace DROP COLUMN domain;

backend/migrations/20221213112420_add_operator.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE workspace_invite DROP COLUMN operator;
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_operator;

backend/migrations/20221213112420_add_operator.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE workspace_invite ADD COLUMN operator BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_operator BOOLEAN DEFAULT false;
+ALTER TABLE completed_job ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 

backend/migrations/20221215101605_share_job_to_owner.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN visible_to_owner;
+ALTER TABLE completed_job DROP COLUMN visible_to_owner;
+
+DROP POLICY see_own_path ON queue;

backend/migrations/20221215101605_share_job_to_owner.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+ALTER TABLE completed_job ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+
+CREATE POLICY see_own_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'u' AND SPLIT_PART(queue.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'g' AND SPLIT_PART(queue.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_own_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'u' AND SPLIT_PART(completed_job.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'g' AND SPLIT_PART(completed_job.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20221219032311_folders.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221219032311_folders.up.sql

@@ -0,0 +1,19 @@
+-- Add up migration script here
+ALTER TABLE app_version
+RENAME COLUMN flow_id TO app_id;
+
+CREATE TABLE folder (
+    name VARCHAR(255),
+    workspace_id VARCHAR(50) NOT NULL,
+    display_name VARCHAR(100) NOT NULL,
+    owners VARCHAR(255)[] NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}',
+    FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE,
+    PRIMARY KEY (workspace_id, name)
+);
+
+CREATE INDEX folder_extra_perms ON folder USING GIN (extra_perms);
+CREATE INDEX folder_owners ON folder USING GIN (owners);
+
+GRANT ALL ON folder TO windmill_user;
+GRANT ALL ON folder TO windmill_admin;

backend/migrations/20221219051235_folders_2.down.sql

@@ -0,0 +1,21 @@
+-- Add down migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        DROP POLICY see_folder_extra_perms_user ON %1$I;
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+
+DROP POLICY see_extra_perms_user ON folder;
+DROP POLICY see_extra_perms_groups ON folder;

backend/migrations/20221219051235_folders_2.up.sql

@@ -0,0 +1,52 @@
+-- Add up migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL
+        USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+        WITH CHECK (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+ALTER TABLE folder ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY see_extra_perms_user ON folder FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')) or (CONCAT('u/', current_setting('session.user')) = ANY(owners)))
+WITH CHECK ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
+
+CREATE POLICY see_extra_perms_groups ON folder FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[] or (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]))))
+WITH CHECK (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
+
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['script', 'flow', 'variable', 'resource', 'schedule'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        ALTER TABLE %1$I DROP CONSTRAINT proper_id;
+        ALTER TABLE %1$I ADD CONSTRAINT proper_id CHECK (path ~ '^[ufg](\/[\w-]+){2,}$');
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;

backend/migrations/20230101074836_sync_from_admin.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230101074836_sync_from_admin.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.57.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230107075643_use_internal_url_sync.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230107075643_use_internal_url_sync.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.57.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230119194229_customer_id.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230119194229_customer_id.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN customer_id VARCHAR(100);
+ALTER TABLE workspace_settings ADD COLUMN plan VARCHAR(40);

backend/migrations/20230126023323_webhook.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230126023323_webhook.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE
+    workspace_settings
+ADD
+    COLUMN webhook text;

backend/oauth_connect.json

@@ -1,96 +1,81 @@
 {
-    "github": {
-        "auth_url": "https://github.com/login/oauth/authorize",
-        "token_url": "https://github.com/login/oauth/access_token",
-        "scopes": [
-            "workflow",
-            "repo"
-        ]
-    },
-    "gitlab": {
-        "auth_url": "https://gitlab.com/oauth/authorize",
-        "token_url": "https://gitlab.com/oauth/token",
-        "scopes": [
-            "api"
-        ]
-    },
-    "bitbucket": {
-        "auth_url": "https://bitbucket.org/site/oauth2/authorize",
-        "token_url": "https://bitbucket.org/site/oauth2/access_token",
-        "scopes": [
-            "repository"
-        ]
-    },
-    "slack": {
-        "auth_url": "https://slack.com/oauth/authorize",
-        "token_url": "https://slack.com/api/oauth.access",
-        "scopes": [
-            "chat:write:user",
-            "users:read",
-            "users:read.email"
-        ]
-    },
-    "gsheets": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/spreadsheets"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gdrive": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/drive"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gmail": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/gmail.send"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcal": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/calendar.events"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "gcloud": {
-        "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
-        "token_url": "https://oauth2.googleapis.com/token",
-        "scopes": [
-            "https://www.googleapis.com/auth/cloud-platform"
-        ],
-        "extra_params": {
-            "access_type": "offline",
-            "consent": "prompt"
-        }
-    },
-    "basecamp": {
-        "auth_url": "https://launchpad.37signals.com/authorization/new",
-        "token_url": "https://launchpad.37signals.com/authorization/token",
-        "scopes": [],
-        "extra_params": {
-            "type": "web_server"
-        }
+  "github": {
+    "auth_url": "https://github.com/login/oauth/authorize",
+    "token_url": "https://github.com/login/oauth/access_token",
+    "scopes": ["workflow", "repo"]
+  },
+  "gitlab": {
+    "auth_url": "https://gitlab.com/oauth/authorize",
+    "token_url": "https://gitlab.com/oauth/token",
+    "scopes": ["api"]
+  },
+  "bitbucket": {
+    "auth_url": "https://bitbucket.org/site/oauth2/authorize",
+    "token_url": "https://bitbucket.org/site/oauth2/access_token",
+    "scopes": ["repository"]
+  },
+  "slack": {
+    "auth_url": "https://slack.com/oauth/authorize",
+    "token_url": "https://slack.com/api/oauth.access",
+    "scopes": ["chat:write:user", "users:read", "users:read.email"]
+  },
+  "gsheets": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/spreadsheets"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
     }
+  },
+  "gdrive": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/drive"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gmail": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/gmail.send"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gcal": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/calendar.events"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "gcloud": {
+    "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
+    "token_url": "https://oauth2.googleapis.com/token",
+    "scopes": ["https://www.googleapis.com/auth/cloud-platform"],
+    "extra_params": {
+      "access_type": "offline",
+      "prompt": "consent"
+    }
+  },
+  "basecamp": {
+    "auth_url": "https://launchpad.37signals.com/authorization/new",
+    "token_url": "https://launchpad.37signals.com/authorization/token",
+    "scopes": [],
+    "extra_params": {
+      "type": "web_server"
+    }
+  },
+  "linkedin": {
+    "auth_url": "https://www.linkedin.com/oauth/v2/authorization",
+    "token_url": "https://www.linkedin.com/oauth/v2/accessToken",
+    "scopes": ["w_member_social", "r_liteprofile", "r_emailaddress"],
+    "req_body_auth": true
+  }
 }

backend/oauth_login.json

@@ -2,16 +2,25 @@
   "github": {
     "auth_url": "https://github.com/login/oauth/authorize",
     "token_url": "https://github.com/login/oauth/access_token",
+    "userinfo_url": "https://api.github.com/user",
     "scopes": ["user:email"]
   },
   "gitlab": {
     "auth_url": "https://gitlab.com/oauth/authorize",
     "token_url": "https://gitlab.com/oauth/token",
+    "userinfo_url": "https://gitlab.com/api/v4/user",
     "scopes": ["read_user"]
   },
   "google": {
     "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
     "token_url": "https://oauth2.googleapis.com/token",
+    "userinfo_url": "https://www.googleapis.com/oauth2/v1/userinfo?alt=json",
     "scopes": ["https://www.googleapis.com/auth/userinfo.email"]
+  },
+  "microsoft": {
+    "auth_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+    "token_url": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+    "userinfo_url": "https://graph.microsoft.com/oidc/userinfo",
+    "scopes": ["openid", "profile", "email"]
   }
 }

backend/parsers/windmill-parser-bash/Cargo.toml

@@ -15,4 +15,5 @@ phf.workspace = true
 unicode-general-category.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
-regex.workspace = true
+regex.workspace = true
+lazy_static.workspace = true

backend/parsers/windmill-parser-go/src/lib.rs

@@ -99,6 +99,11 @@ fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option<String>, Typ) {
                 Typ::Object(typs),
             )
         }
+        Expr::InterfaceType(_) => (Some("interface{}".to_string()), Typ::Object(vec![])),
+        Expr::MapType(_) => (
+            Some("map[string]interface{}".to_string()),
+            Typ::Object(vec![]),
+        ),
         _ => (None, Typ::Unknown),
     }
 }
@@ -122,7 +127,7 @@ package main
 
 import "fmt"
 
-func main(x int, y string, z bool, l []string, o struct { Name string `json:"name"` }) {
+func main(x int, y string, z bool, l []string, o struct { Name string `json:"name"` }, n interface{}, m map[string]interface{}) {
     fmt.Println("hello world")
 }    
 
@@ -172,6 +177,20 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
                         default: None,
                         has_default: false
                     },
+                    Arg {
+                        otyp: Some("interface{}".to_string()),
+                        name: "n".to_string(),
+                        typ: Typ::Object(vec![]),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: Some("map[string]interface{}".to_string()),
+                        name: "m".to_string(),
+                        typ: Typ::Object(vec![]),
+                        default: None,
+                        has_default: false
+                    },
                 ]
             }
         );

backend/parsers/windmill-parser-py/Cargo.toml

@@ -17,3 +17,4 @@ itertools.workspace = true
 regex.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
+lazy_static.workspace = true

backend/parsers/windmill-parser-py/src/lib.rs

@@ -9,6 +9,7 @@
 use std::collections::HashMap;
 
 use itertools::Itertools;
+use lazy_static::lazy_static;
 use phf::phf_map;
 use regex::Regex;
 
@@ -103,23 +104,20 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
                             "bytes" => Typ::Bytes,
                             "datetime" => Typ::Datetime,
                             "datetime.datetime" => Typ::Datetime,
-                            _ => Typ::Unknown,
+                            "Sql" | "sql" => Typ::Sql,
+                            _ => Typ::Resource(id),
                         },
                         _ => Typ::Unknown,
                     });
+
                     if typ == Typ::Unknown
                         && default.is_some()
                         && default != Some(json!(FUNCTION_CALL))
                     {
                         typ = json_to_typ(default.as_ref().unwrap());
                     }
-                    Arg {
-                        otyp: None,
-                        name: x.arg,
-                        typ: typ,
-                        has_default: default.is_some(),
-                        default,
-                    }
+
+                    Arg { otyp: None, name: x.arg, typ, has_default: default.is_some(), default }
                 })
                 .collect(),
         })
@@ -177,7 +175,13 @@ fn constant_to_value(c: &Constant) -> serde_json::Value {
 
 static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
     "psycopg2" => "psycopg2-binary",
-    "git" => "GitPython"
+    "psycopg" => "psycopg[binary, pool]",
+    "yaml" => "pyyaml",
+    "git" => "GitPython",
+    "u" => "requests",
+    "f" => "requests",
+    "shopify" => "ShopifyAPI",
+    "seleniumwire" => "selenium-wire",
 };
 
 fn replace_import(x: String) -> String {
@@ -188,17 +192,20 @@ fn replace_import(x: String) -> String {
         .to_string()
 }
 
+lazy_static! {
+    static ref RE: Regex = Regex::new(r"^\#(\S+)$").unwrap();
+}
+
 pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
     let find_requirements = code
         .lines()
         .find_position(|x| x.starts_with("#requirements:"));
-    let re = Regex::new(r"^\#(\S+)$").unwrap();
     if let Some((pos, _)) = find_requirements {
         let lines = code
             .lines()
             .skip(pos + 1)
             .map_while(|x| {
-                re.captures(x)
+                RE.captures(x)
                     .map(|x| x.get(1).unwrap().as_str().to_string())
             })
             .collect();
@@ -325,9 +332,11 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
 
 import os
 
+postgresql = dict
 def main(test1: str,
     name: datetime.datetime = datetime.now(),
-    byte: bytes = bytes(1)):
+    byte: bytes = bytes(1),
+    resource: postgresql = \"$res:g/all/resource\"):
 
 	print(f\"Hello World and a warm welcome especially to {name}\")
 	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
@@ -361,6 +370,13 @@ def main(test1: str,
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
                         has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "resource".to_string(),
+                        typ: Typ::Resource("postgresql".to_string()),
+                        default: Some(json!("$res:g/all/resource")),
+                        has_default: true
                     }
                 ]
             }
@@ -430,7 +446,7 @@ def main():
 
 ";
         let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
+        // println!("{}", serde_json::to_string(&r)?);
         assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
         Ok(())
     }

backend/src/ee.rs

@@ -0,0 +1,24 @@
+#[cfg(feature = "enterprise")]
+use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
+#[cfg(feature = "enterprise")]
+use sha2::Sha256;
+
+#[cfg(feature = "enterprise")]
+pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
+    if let Some(license_key) = license_key {
+        let splitted_lk = license_key
+            .split_once(".")
+            .expect("license_key can be splitted with a .");
+
+        let pub_key = rsa::RsaPublicKey::from_public_key_der(
+    &base64::decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
+        let msg = base64::decode(splitted_lk.0)?;
+        let signature = base64::decode(splitted_lk.1)?;
+        rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
+            .verify(&msg, &rsa::pss::Signature::from(signature))
+            .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
+    } else {
+        panic!("License key is required for the enterprise edition");
+    }
+    Ok(())
+}

backend/src/main.rs

@@ -8,18 +8,21 @@
 
 use std::net::SocketAddr;
 
+use git_version::git_version;
 use sqlx::{Pool, Postgres};
 use windmill_common::utils::rd_string;
 use windmill_worker::WorkerConfig;
 
+const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
+
+mod ee;
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     dotenv::dotenv().ok();
 
     windmill_common::tracing_init::initialize_tracing();
 
-    let db = windmill_common::connect_db().await?;
-
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
@@ -40,6 +43,8 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false);
 
+    let db = windmill_common::connect_db(server_mode).await?;
+
     if server_mode {
         windmill_api::migrate_db(&db).await?;
     }
@@ -47,11 +52,10 @@ async fn main() -> anyhow::Result<()> {
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
     let shutdown_signal = windmill_common::shutdown_signal(tx);
 
-    let base_internal_url =
-        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
-
     let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
+    let base_internal_url =
+        std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
     let timeout = std::env::var("TIMEOUT")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
@@ -82,17 +86,39 @@ async fn main() -> anyhow::Result<()> {
                 let disable_nsjail = std::env::var("DISABLE_NSJAIL")
                     .ok()
                     .and_then(|x| x.parse::<bool>().ok())
-                    .unwrap_or(false);
+                    .unwrap_or(true);
                 let keep_job_dir = std::env::var("KEEP_JOB_DIR")
                     .ok()
                     .and_then(|x| x.parse::<bool>().ok())
                     .unwrap_or(false);
+                let license_key = std::env::var("LICENSE_KEY").ok();
+                let sync_bucket = std::env::var("S3_CACHE_BUCKET")
+                    .ok()
+                    .map(|e| Some(e))
+                    .unwrap_or(None);
+
+                #[cfg(feature = "enterprise")]
+                tracing::info!(
+                        "
+##############################
+Windmill Enterprise Edition {GIT_VERSION} LICENSE_KEY: {license_key:?}, S3_CACHE_BUCKET: {sync_bucket:?}
+##############################"
+                    );
+
+                #[cfg(not(feature = "enterprise"))]
+                tracing::info!(
+                    "
+##############################
+Windmill Community Edition {GIT_VERSION}
+##############################"
+                );
 
                 tracing::info!(
                     "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
                      {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
                      {timeout}, KEEP_JOB_DIR: {keep_job_dir}"
                 );
+
                 run_workers(
                     db.clone(),
                     addr,
@@ -107,6 +133,8 @@ async fn main() -> anyhow::Result<()> {
                         keep_job_dir,
                     },
                     rx.resubscribe(),
+                    sync_bucket,
+                    license_key,
                 )
                 .await?;
             }
@@ -160,7 +188,23 @@ pub async fn run_workers(
     sleep_queue: u64,
     worker_config: WorkerConfig,
     rx: tokio::sync::broadcast::Receiver<()>,
+    mut periodic_script: Option<String>,
+    license_key: Option<String>,
 ) -> anyhow::Result<()> {
+    #[cfg(feature = "enterprise")]
+    ee::verify_license_key(license_key)?;
+
+    #[cfg(not(feature = "enterprise"))]
+    if license_key.is_some() {
+        panic!("License key is required ONLY for the enterprise edition");
+    }
+    #[cfg(not(feature = "enterprise"))]
+    if !worker_config.disable_nsjail {
+        tracing::warn!(
+            "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
+        );
+    }
+
     let instance_name = rd_string(5);
     let monitor = tokio_metrics::TaskMonitor::new();
 
@@ -180,6 +224,7 @@ pub async fn run_workers(
         let ip = ip.clone();
         let rx = rx.resubscribe();
         let worker_config = worker_config.clone();
+        let wp = periodic_script.take();
         handles.push(tokio::spawn(monitor.instrument(async move {
             tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
@@ -192,6 +237,7 @@ pub async fn run_workers(
                 &ip,
                 sleep_queue,
                 worker_config,
+                wp,
                 rx,
             )
             .await

backend/tests/fixtures/base.sql

@@ -2,8 +2,8 @@
 -- https://docs.rs/sqlx/latest/sqlx/attr.test.html
 
 INSERT INTO workspace
-            (id,               name,             owner,       domain)
-     VALUES ('test-workspace', 'test-workspace', 'test-user', null);
+            (id,               name,             owner)
+     VALUES ('test-workspace', 'test-workspace', 'test-user');
 
 INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
 	('test-workspace', 'test@windmill.dev', 'test-user', true, 'Admin');
@@ -11,6 +11,8 @@ INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
 INSERT INTO workspace_key(workspace_id, kind, key) VALUES
 	('test-workspace', 'cloud', 'test-key');
 
+insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
+
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 
@@ -45,4 +47,4 @@ EXECUTE FUNCTION "notify_queue" ();
  AFTER UPDATE ON "queue"
     FOR EACH ROW
             WHEN (NEW.flow_status IS DISTINCT FROM OLD.flow_status)
-EXECUTE FUNCTION "notify_queue" ();
+EXECUTE FUNCTION "notify_queue" ();

backend/tests/worker.rs

@@ -162,6 +162,7 @@ mod suspend_resume {
     fn flow() -> FlowValue {
         serde_json::from_value(serde_json::json!({
                 "modules": [{
+                    "id": "a",
                     "input_transform": {
                         "n": { "type": "javascript", "expr": "flow_input.n", },
                         "port": { "type": "javascript", "expr": "flow_input.port", },
@@ -185,7 +186,7 @@ mod suspend_resume {
                                 const secret = await r.text();\
                                 console.log('Secret: ' + secret + ' ' + job + ' ' + token);\
                                 const r2 = await fetch(
-                                    `http://localhost:${port}/api/w/test-workspace/jobs/${op}/${job}/0/${secret}?approver=ruben`,\
+                                    `http://localhost:${port}/api/w/test-workspace/jobs_u/${op}/${job}/0/${secret}?approver=ruben`,\
                                     {\
                                         method: 'POST',\
                                         body: JSON.stringify('from job'),\
@@ -200,8 +201,9 @@ mod suspend_resume {
                         "required_events": 1
                     },
                 }, {
+                    "id": "b",
                     "input_transform": {
-                        "n": { "type": "javascript", "expr": "previous_result", },
+                        "n": { "type": "javascript", "expr": "results.a", },
                         "resume": { "type": "javascript", "expr": "resume", },
                         "resumes": { "type": "javascript", "expr": "resumes", },
                     },
@@ -215,7 +217,7 @@ mod suspend_resume {
                     },
                 }, {
                     "input_transform": {
-                        "last": { "type": "javascript", "expr": "previous_result", },
+                        "last": { "type": "javascript", "expr": "results.b", },
                         "resume": { "type": "javascript", "expr": "resume", },
                         "resumes": { "type": "javascript", "expr": "resumes", },
                     },
@@ -274,7 +276,7 @@ mod suspend_resume {
 
                 /* ImZyb20gdGVzdCIK = base64 "from test" */
                 reqwest::get(format!(
-                    "http://localhost:{port}/api/w/test-workspace/jobs/resume/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK&approver=ruben"
+                    "http://localhost:{port}/api/w/test-workspace/jobs_u/resume/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK&approver=ruben"
                 ))
                 .await
                 .unwrap()
@@ -331,7 +333,7 @@ mod suspend_resume {
         server.close().await.unwrap();
 
         assert_eq!(
-            json!({"error": "Job canceled: approval request disapproved by ruben" }),
+            json!({"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by ruben", "canceler": "ruben"}}),
             result
         );
     }
@@ -377,7 +379,7 @@ mod suspend_resume {
 
                 /* ImZyb20gdGVzdCIK = base64 "from test" */
                 reqwest::get(format!(
-                    "http://localhost:{port}/api/w/test-workspace/jobs/cancel/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK"
+                    "http://localhost:{port}/api/w/test-workspace/jobs_u/cancel/{second}/0/{secret}?payload=ImZyb20gdGVzdCIK"
                 ))
                 .await
                 .unwrap()
@@ -393,7 +395,7 @@ mod suspend_resume {
         let result = completed_job(flow, &db).await.result.unwrap();
 
         assert_eq!(
-            json!({"error": "Job canceled: approval request disapproved by unknown" }),
+            json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
             result
         );
     }
@@ -460,7 +462,7 @@ export async function main(index, port) {
     const buf = new Uint8Array([0]);
     const sock = await Deno.connect({ port });
     await sock.write(new Uint8Array([index]));
-    if (await sock.read(buf) != 1) throw "read";
+    if (await sock.read(buf) != 1) throw Error("read");
     return buf[0];
 }
             "#
@@ -478,13 +480,14 @@ def main(last, port):
     fn flow_forloop_retry() -> FlowValue {
         serde_json::from_value(serde_json::json!({
             "modules": [{
+                "id": "a",
                 "value": {
                     "type": "forloopflow",
-                    "iterator": { "type": "javascript", "expr": "result.items" },
+                    "iterator": { "type": "javascript", "expr": "flow_input.items" },
                     "skip_failures": false,
                     "modules": [{
                         "input_transform": {
-                            "index": { "type": "javascript", "expr": "previous_result.iter.index" },
+                            "index": { "type": "javascript", "expr": "flow_input.iter.index" },
                             "port": { "type": "javascript", "expr": "flow_input.port" },
                         },
                         "value": {
@@ -497,7 +500,7 @@ def main(last, port):
                 "retry": { "constant": { "attempts": 2, "seconds": 0 } },
             }, {
                 "input_transform": {
-                    "last": { "type": "javascript", "expr": "previous_result" },
+                    "last": { "type": "javascript", "expr": "results.a" },
                     "port": { "type": "javascript", "expr": "flow_input.port" },
                 },
                 "value": {
@@ -578,10 +581,17 @@ def main(last, port):
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
-        assert!(result[1]["error"]
-            .as_str()
-            .unwrap()
-            .contains(r#"Uncaught (in promise) "read""#));
+
+        assert!(
+            result[1]["error"]
+                .as_object()
+                .unwrap()
+                .get("message")
+                .unwrap()
+                .as_str()
+                .unwrap()
+                == "read"
+        );
     }
 
     #[sqlx::test(fixtures("base"))]
@@ -603,16 +613,19 @@ def main(last, port):
         .into_iter()
         .unzip::<_, _, Vec<_>, Vec<_>>();
         let server = Server::start(responses).await;
-        let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
+        let job = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
             .arg("items", json!(["unused", "unused", "unused"]))
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
-            .await
-            .result
-            .unwrap();
+            .await;
 
+        let result = job.result.unwrap();
         assert_eq!(server.close().await, attempts);
         assert!(result["error"]
+            .as_object()
+            .unwrap()
+            .get("message")
+            .unwrap()
             .as_str()
             .unwrap()
             .contains("index out of range"));
@@ -682,9 +695,7 @@ def main(error, port):
             result,
             json!({
                 "recv": 42,
-                "from failure module": {
-                    "error": "Error during execution of the script:\n\nTraceback (most recent call last):\n  File \"/tmp/main.py\", line 14, in <module>\n    res = inner_script.main(**kwargs)\n  File \"/tmp/inner.py\", line 5, in main\n    return sock.recv(1)[0]\nIndexError: index out of range",
-                }
+                "from failure module": {"error": {"name": "IndexError", "stack": "  File \"/tmp/inner.py\", line 5, in main\n    return sock.recv(1)[0]\n", "message": "index out of range"}},
             })
         );
     }
@@ -706,7 +717,7 @@ async fn test_iteration(db: Pool<Postgres>) {
                     "input_transform": {
                         "n": {
                             "type": "javascript",
-                            "expr": "previous_result.iter.value",
+                            "expr": "flow_input.iter.value",
                         },
                     },
                     "value": {
@@ -737,9 +748,13 @@ async fn test_iteration(db: Pool<Postgres>) {
         .unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
+        .as_object()
+        .unwrap()
+        .get("message")
+        .unwrap()
         .as_str()
         .unwrap()
-        .contains("StopIteration: 2"));
+        .contains("2"));
 }
 
 #[sqlx::test(fixtures("base"))]
@@ -759,7 +774,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
                     "input_transform": {
                         "n": {
                             "type": "javascript",
-                            "expr": "previous_result.iter.value",
+                            "expr": "flow_input.iter.value",
                         },
                     },
                     "value": {
@@ -790,9 +805,13 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
     let result = job.result.unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
+        .as_object()
+        .unwrap()
+        .get("message")
+        .unwrap()
         .as_str()
         .unwrap()
-        .contains("StopIteration: 2"));
+        .contains("2"));
 }
 
 struct RunJob {
@@ -819,14 +838,17 @@ impl RunJob {
             tx,
             "test-workspace",
             payload,
-            Some(args),
+            args,
             /* user */ "test-user",
+            /* email  */ "test@windmill.dev",
             /* permissioned_as */ "u/admin".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
             /* is_flow_step */ false,
             /* running */ false,
+            None,
+            true,
         )
         .await
         .expect("push has to succeed");
@@ -930,6 +952,7 @@ fn spawn_test_worker(
             ip,
             sleep_queue,
             worker_config,
+            None,
             rx,
         )
         .await
@@ -1007,6 +1030,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                         language: ScriptLang::Deno,
                         content: numbers.to_string(),
                         path: None,
+                        lock: None,
                     },
                     input_transforms: Default::default(),
                     stop_after_if: Default::default(),
@@ -1027,13 +1051,14 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                                 input_transforms: [(
                                     "n".to_string(),
                                     InputTransform::Javascript {
-                                        expr: "previous_result.iter.value".to_string(),
+                                        expr: "flow_input.iter.value".to_string(),
                                     },
                                 )]
                                 .into(),
                                 language: ScriptLang::Deno,
                                 content: doubles.to_string(),
                                 path: None,
+                                lock: None,
                             },
                             input_transforms: Default::default(),
                             stop_after_if: Default::default(),
@@ -1068,6 +1093,42 @@ async fn test_deno_flow(db: Pool<Postgres>) {
     }
 }
 
+#[sqlx::test(fixtures("base"))]
+async fn test_identity(db: Pool<Postgres>) {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await;
+
+    let flow: FlowValue = serde_json::from_value(serde_json::json!({
+        "modules": [{
+                "value": {
+                    "type": "rawscript",
+                    "language": "python3",
+                    "content": "def main(): return 42",
+                }}, {
+                    "value": {
+                        "type": "identity",
+                    },
+                }, {
+                    "value": {
+                        "type": "identity",
+                    },
+                }, {
+                    "value": {
+                        "type": "identity",
+                    },
+                }],
+    }))
+    .unwrap();
+
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+        .run_until_complete(&db, server.addr.port())
+        .await
+        .result
+        .unwrap();
+    assert_eq!(result, serde_json::json!(42));
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1075,7 +1136,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
     let server = ApiServer::start(db.clone()).await;
 
     let write_file = r#"export async function main(loop: boolean, i: number, path: string) {  
-            await Deno.writeTextFile(`/shared/${path}`, `${loop} ${i}`);
+            await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`);
         }"#
     .to_string();
 
@@ -1085,20 +1146,21 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     id: "a".to_string(),
                     value: FlowModuleValue::RawScript {
                         input_transforms: [
-                        (
-                            "loop".to_string(),
-                            InputTransform::Static { value: json!(false) },
-                        ),
-                        ("i".to_string(), InputTransform::Static { value: json!(1) }),
-                        (
-                            "path".to_string(),
-                            InputTransform::Static { value: json!("outer.txt") },
-                        ),
-                    ]
-                    .into(),
+                            (
+                                "loop".to_string(),
+                                InputTransform::Static { value: json!(false) },
+                            ),
+                            ("i".to_string(), InputTransform::Static { value: json!(1) }),
+                            (
+                                "path".to_string(),
+                                InputTransform::Static { value: json!("outer.txt") },
+                            ),
+                        ]
+                        .into(),
                         language: ScriptLang::Deno,
                         content: write_file.clone(),
                         path: None,
+                        lock: None,
                     },
                     input_transforms: Default::default(),
                     stop_after_if: Default::default(),
@@ -1120,7 +1182,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 (
                                     "i".to_string(),
                                     InputTransform::Javascript {
-                                        expr: "previous_result.iter.value".to_string(),
+                                        expr: "flow_input.iter.value".to_string(),
                                     },
                                 ),
                                 (
@@ -1138,6 +1200,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     language: ScriptLang::Deno,
                                     content: write_file,
                                     path: None,
+                                    lock: None,
                                 },
                                 stop_after_if: Default::default(),
                                 summary: Default::default(),
@@ -1158,10 +1221,11 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     .into(),
                                     language: ScriptLang::Deno,
                                     content: r#"export async function main(path: string, path2: string) {  
-                                        return await Deno.readTextFile(`/shared/${path}`) + "," + await Deno.readTextFile(`/shared/${path2}`);
+                                        return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`);
                                     }"#
                                     .to_string(),
                                     path: None,
+                                    lock: None,
                                 },
                                 input_transforms: [].into(),
                                 stop_after_if: Default::default(),
@@ -1184,9 +1248,10 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     id: "c".to_string(),
                     value: FlowModuleValue::RawScript {
                         input_transforms: [
+
                         (
                             "loops".to_string(),
-                            InputTransform::Javascript { expr: "previous_result".to_string() },
+                            InputTransform::Javascript { expr: "results.b".to_string() },
                         ),
                         (
                             "path".to_string(),
@@ -1200,10 +1265,11 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     .into(),
                         language: ScriptLang::Deno,
                         content: r#"export async function main(path: string, loops: string[], path2: string) {
-                            return await Deno.readTextFile(`/shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`/shared/${path2}`);
+                            return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`);
                         }"#
                         .to_string(),
                         path: None,
+                        lock: None,
                     },
                     input_transforms: [].into(),
                     stop_after_if: Default::default(),
@@ -1255,7 +1321,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
                                         "branches": [{"modules": [                {
                                             "id": "d",
                                             "value": {
-                                                "input_transforms": {"v": {"type": "javascript", "expr": "result_by_id(\"a\")"}},
+                                                "input_transforms": {"v": {"type": "javascript", "expr": "results.a"}},
                                                 "type": "rawscript",
                                                 "language": "deno",
                                                 "content": "export function main(v){ return v }",
@@ -1306,7 +1372,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
             {
                 "id": "b",
                 "value": {
-                    "input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } },
+                    "input_transforms": { "n": { "type": "javascript", "expr": "results.a" } },
                     "type": "rawscript",
                     "language": "python3",
                     "content": "def main(n): return f'last step saw {n}'",
@@ -1364,7 +1430,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
             {
                 "id": "c",
                 "value": {
-                    "input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } },
+                    "input_transforms": { "n": { "type": "javascript", "expr": "results.a" } },
                     "type": "rawscript",
                     "language": "python3",
                     "content": "def main(n): return f'last step saw {n}'",
@@ -1425,7 +1491,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
                         "input_transform": {
                             "n": {
                                 "type": "javascript",
-                                "expr": "previous_result.iter.value",
+                                "expr": "flow_input.iter.value",
                             },
                         },
                     }],
@@ -1435,7 +1501,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
     }))
     .unwrap();
 
-    for i in 0..50 {
+    for i in 0..10 {
         println!("python flow iteration: {}", i);
         let result = run_job_in_new_worker_until_complete(
             &db,
@@ -1504,6 +1570,7 @@ func main(derp string) (string, error) {
     let result = RunJob::from(JobPayload::Code(RawCode {
         content,
         path: None,
+        lock: None,
         language: ScriptLang::Go,
     }))
     .arg("derp", json!("world"))
@@ -1530,6 +1597,7 @@ echo "hello $msg"
     let job = RunJob::from(JobPayload::Code(RawCode {
         content,
         path: None,
+        lock: None,
         language: ScriptLang::Bash,
     }))
     .arg("msg", json!("world"))
@@ -1551,7 +1619,12 @@ def main():
         "#
     .to_owned();
 
-    let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 });
+    let job = JobPayload::Code(RawCode {
+        content,
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+    });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
@@ -1576,7 +1649,12 @@ def main():
         "#
     .to_owned();
 
-    let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 });
+    let job = JobPayload::Code(RawCode {
+        content,
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+    });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
@@ -1600,7 +1678,12 @@ def main():
         "#
     .to_owned();
 
-    let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 });
+    let job = JobPayload::Code(RawCode {
+        content,
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+    });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
@@ -1619,6 +1702,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(serde_json::json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "forloopflow",
                     "iterator": { "type": "static", "value": [] },
@@ -1628,7 +1712,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
                                 "input_transform": {
                                     "n": {
                                         "type": "javascript",
-                                        "expr": "previous_result.iter.value",
+                                        "expr": "flow_input.iter.value",
                                     },
                                 },
                                 "type": "rawscript",
@@ -1644,7 +1728,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
                     "input_transform": {
                         "items": {
                             "type": "javascript",
-                            "expr": "previous_result",
+                            "expr": "results.a",
                         },
                     },
                     "type": "rawscript",
@@ -1700,7 +1784,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
 
     assert_eq!(
         job.result.unwrap(),
-        serde_json::json!({"error":"Expected an array value, found: {}"})
+        serde_json::json!( {"error":  {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
     );
 }
 
@@ -1721,7 +1805,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
                             "input_transform": {
                                 "n": {
                                     "type": "javascript",
-                                    "expr": "previous_result.iter.value",
+                                    "expr": "flow_input.iter.value",
                                 },
                             },
                             "value": {
@@ -1754,6 +1838,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(serde_json::json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "forloopflow",
                     "iterator": { "type": "static", "value": [2,3,4] },
@@ -1762,7 +1847,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
                             "input_transform": {
                                 "n": {
                                     "type": "javascript",
-                                    "expr": "previous_result.iter.value",
+                                    "expr": "flow_input.iter.value",
                                 },
                             },
                             "value": {
@@ -1778,7 +1863,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
                 "input_transform": {
                     "items": {
                         "type": "javascript",
-                        "expr": "previous_result",
+                        "expr": "results.a",
                     },
                 },
                 "value": {
@@ -1800,12 +1885,13 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!(9));
 }
 
-fn module_add_item_to_list(i: i32) -> serde_json::Value {
+fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
     json!({
+        "id": format!("id_{}", i.to_string().replace("-", "_")),
         "input_transform": {
             "array": {
                 "type": "javascript",
-                "expr": "previous_result",
+                "expr": format!("results.{id}"),
             },
             "i": {
                 "type": "static",
@@ -1840,6 +1926,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -1849,7 +1936,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
             {
                 "value": {
                     "branches": [],
-                    "default": [module_add_item_to_list(2)],
+                    "default": [module_add_item_to_list(2, "a")],
                     "type": "branchone",
                 }
             },
@@ -1884,8 +1971,8 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
             },
             {
                 "value": {
-                    "branches": [{"expr": "previous_result[0] == 1 && result_by_id(\"a\")[0] == 1", "modules": [module_add_item_to_list(3)]}],
-                    "default": [module_add_item_to_list(2)],
+                    "branches": [{"expr": "results.a[0] == 1", "modules": [module_add_item_to_list(3, "a")]}],
+                    "default": [module_add_item_to_list(2, "a")],
                     "type": "branchone",
                 }
             },
@@ -1911,6 +1998,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -1920,8 +2008,8 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
             {
                 "value": {
                     "branches": [
-                        {"modules": [module_add_item_to_list(2)]},
-                        {"modules": [module_add_item_to_list(3)]}],
+                        {"modules": [module_add_item_to_list(2, "a")]},
+                        {"modules": [module_add_item_to_list(3, "a")]}],
                     "type": "branchall",
                     "parallel": true,
                 }
@@ -1948,6 +2036,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -1957,8 +2046,8 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
             {
                 "value": {
                     "branches": [
-                        {"modules": [module_add_item_to_list(2)]},
-                        {"modules": [module_add_item_to_list(3)]}],
+                        {"modules": [module_add_item_to_list(2, "a")]},
+                        {"modules": [module_add_item_to_list(3, "a")]}],
                     "type": "branchall",
                 }
             },
@@ -1984,6 +2073,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -1994,7 +2084,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
                 "value": {
                     "branches": [
                         {"modules": [module_failure()], "skip_failure": false},
-                        {"modules": [module_add_item_to_list(3)]}],
+                        {"modules": [module_add_item_to_list(3, "a")]}],
                     "type": "branchall",
                 }
             },
@@ -2010,12 +2100,13 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
 
     assert_eq!(
         result,
-        serde_json::json!([{"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n                              ^\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1"}, [1,3]])
+        serde_json::json!([{"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1", "message": "failure"}}, [1,3]])
     );
 
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -2026,7 +2117,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
                 "value": {
                     "branches": [
                         {"modules": [module_failure()], "skip_failure": true},
-                        {"modules": [module_add_item_to_list(2)]}
+                        {"modules": [module_add_item_to_list(2, "a")]}
                 ],
                     "type": "branchall",
                 }
@@ -2043,7 +2134,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
 
     assert_eq!(
         result,
-        serde_json::json!([{"error": "Error during execution of the script:\n\nerror: Uncaught (in promise) Error: failure\nexport function main(){ throw Error('failure') }\n                              ^\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1"}, [1, 2]])
+        serde_json::json!([ {"error": {"name": "Error", "stack": "Error: failure\n    at main (file:///tmp/inner.ts:1:31)\n    at run (file:///tmp/main.ts:9:26)\n    at file:///tmp/main.ts:14:1", "message": "failure"}}, [1, 2]])
     );
 }
 
@@ -2056,14 +2147,16 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
                     "content": "export function main(){ return [] }",
                 }
             },
-            module_add_item_to_list(1),
+            module_add_item_to_list(1, "a"),
             {
+                "id": "b",
                 "value": {
                     "branches": [
                         {
@@ -2079,17 +2172,17 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
                                             "expr": "false",
                                             "modules": []
                                         }],
-                                    "default": [module_add_item_to_list(2)],
+                                    "default": [module_add_item_to_list(2, "id_1")],
                                     "type": "branchone",
                                 }
                             }]
                         },
                     ],
-                    "default": [module_add_item_to_list(-4)],
+                    "default": [module_add_item_to_list(-4, "id_1")],
                     "type": "branchone",
                 }
             },
-            module_add_item_to_list(3),
+            module_add_item_to_list(3, "b"),
         ],
     }))
     .unwrap();
@@ -2112,6 +2205,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
     let flow: FlowValue = serde_json::from_value(json!({
         "modules": [
             {
+                "id": "a",
                 "value": {
                     "type": "rawscript",
                     "language": "deno",
@@ -2122,24 +2216,27 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
                 "value": {
                     "branches": [
                         {
-                            "modules": [                {
+                            "modules": [
+                                    {
+                                "id": "b",
                                 "value": {
                                     "branches": [
-                                        {"modules": [module_add_item_to_list(2)]},
-                                        {"modules": [module_add_item_to_list(3)]}],
+                                        {"modules": [module_add_item_to_list(2, "a")]},
+                                        {"modules": [module_add_item_to_list(3, "a")]}],
                                     "type": "branchall",
                                 }
                             }, {
                                 "value": {
                                     "branches": [
-                                        {"modules": [module_add_item_to_list(4)]},
-                                        {"modules": [module_add_item_to_list(5)]}],
+                                        {"modules": [module_add_item_to_list(4, "b")]},
+                                        {"modules": [module_add_item_to_list(5, "b")]}],
                                     "type": "branchall",
                                 }
                             }
                                     ]
                         },
-                        {"modules": [module_add_item_to_list(6)]}],
+                        {"modules": [module_add_item_to_list(6, "a")]}],
+                        // "parallel": false,
                     "type": "branchall",
                 }
             },
@@ -2153,6 +2250,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
         .result
         .unwrap();
 
+    println!("{:#?}", result);
     assert_eq!(
         result,
         serde_json::json!([[[[1, 2], [1, 3], 4], [[1, 2], [1, 3], 5]], [1, 6]])
@@ -2167,34 +2265,36 @@ async fn test_failure_module(db: Pool<Postgres>) {
 
     let flow: FlowValue = serde_json::from_value(serde_json::json!({
             "modules": [{
-                "input_transform": {
-                    "l": { "type": "javascript", "expr": "[]", },
-                    "n": { "type": "javascript", "expr": "flow_input.n", },
-                },
+                "id": "a",
                 "value": {
+                    "input_transform": {
+                        "l": { "type": "javascript", "expr": "[]", },
+                        "n": { "type": "javascript", "expr": "flow_input.n", },
+                    },
                     "type": "rawscript",
                     "language": "deno",
-                    "content": "export function main(n, l) { if (n == 0) throw l; return { l: [...l, 0] } }",
+                    "content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }",
                 },
             }, {
-                "input_transform": {
-                    "l": { "type": "javascript", "expr": "previous_result.l", },
-                    "n": { "type": "javascript", "expr": "flow_input.n", },
-                },
+                "id": "b",
                 "value": {
+                    "input_transform": {
+                        "l": { "type": "javascript", "expr": "results.a.l", },
+                        "n": { "type": "javascript", "expr": "flow_input.n", },
+                    },
                     "type": "rawscript",
                     "language": "deno",
-                    "content": "export function main(n, l) { if (n == 1) throw l; return { l: [...l, 1] } }",
+                    "content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }",
                 },
             }, {
-                "input_transform": {
-                    "l": { "type": "javascript", "expr": "previous_result.l", },
-                    "n": { "type": "javascript", "expr": "flow_input.n", },
-                },
                 "value": {
+                    "input_transform": {
+                        "l": { "type": "javascript", "expr": "results.b.l", },
+                        "n": { "type": "javascript", "expr": "flow_input.n", },
+                    },
                     "type": "rawscript",
                     "language": "deno",
-                    "content": "export function main(n, l) { if (n == 2) throw l; return { l: [...l, 2] } }",
+                    "content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }",
                 },
             }],
             "failure_module": {
@@ -2214,10 +2314,15 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .await
         .result
         .unwrap();
+
     assert!(result["from failure module"]["error"]
+        .as_object()
+        .unwrap()
+        .get("message")
+        .unwrap()
         .as_str()
         .unwrap()
-        .contains("Uncaught (in promise) []"));
+        .contains("[]"));
 
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .arg("n", json!(1))
@@ -2225,10 +2330,15 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .await
         .result
         .unwrap();
+
     assert!(result["from failure module"]["error"]
+        .as_object()
+        .unwrap()
+        .get("message")
+        .unwrap()
         .as_str()
         .unwrap()
-        .contains("Uncaught (in promise) [ 0 ]"));
+        .contains("[0]"));
 
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .arg("n", json!(2))
@@ -2236,10 +2346,15 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .await
         .result
         .unwrap();
+
     assert!(result["from failure module"]["error"]
+        .as_object()
+        .unwrap()
+        .get("message")
+        .unwrap()
         .as_str()
         .unwrap()
-        .contains("Uncaught (in promise) [ 0, 1 ]"));
+        .contains("[0,1]"));
 
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .arg("n", json!(3))
@@ -2250,24 +2365,143 @@ async fn test_failure_module(db: Pool<Postgres>) {
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
 
-// #[cfg(test)]
-// mod client_test {
-//     use windmill_common::error::to_anyhow;
+#[sqlx::test(fixtures("base"))]
+async fn test_flow_lock_all(db: Pool<Postgres>) {
+    use futures::StreamExt;
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
 
-//     #[tokio::test]
-//     async fn test_rust_client() -> Result<(), Box<dyn std::error::Error>> {
-//         println!(
-//             "{:#?}",
-//             windmill_api_client::create_client(
-//                 "http://windmill.wimill.xyz",
-//                 "XXXXXXXXXXXXXXX".to_string(),
-//             )
-//             .get_variable("demo", "u/ruben/test", Some(true))
-//             .await
-//             .map_err(to_anyhow)
-//             .map(|v| { v.into_inner() })?
-//             .value
-//         );
-//         Ok(())
-//     }
-// }
+    let flow: windmill_api_client::types::OpenFlow = serde_json::from_value(serde_json::json!({
+        "summary": "",
+        "description": "",
+        "value": {
+            "modules": [
+                {
+                    "id": "a",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "import wmill\n\ndef main():\n  return \"Test\"\n",
+                        "language": "python3",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "b",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main() {\n  return \"Hello\"\n}\n",
+                        "language": "deno",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "c",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "package inner\n\nimport (\n\t\"fmt\"\n\t\"rsc.io/quote\"\n  wmill \"github.com/windmill-labs/windmill-go-client\"\n)\n\n// the main must return (interface{}, error)\n\nfunc main() (interface{}, error) {\n\tfmt.Println(\"Hello, World\")\n  // v, _ := wmill.GetVariable(\"g/all/pretty_secret\")\n  return \"Test\"\n}\n",
+                        "language": "go",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                },
+                {
+                    "id": "d",
+                    "value": {
+                        "lock": null,
+                        "path": null,
+                        "type": "rawscript",
+                        "content": "\n# the last line of the stdout is the return value\necho \"Hello $msg\"\n",
+                        "language": "bash",
+                        "input_transforms": {}
+                    },
+                    "summary": null,
+                    "stop_after_if": null,
+                    "input_transforms": {}
+                }
+            ],
+            "failure_module": null
+        },
+        "schema": {
+            "type": "object",
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "required": [],
+            "properties": {}
+        }
+    }))
+    .unwrap();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_owned(),
+    );
+    client
+        .create_flow(
+            "test-workspace",
+            &windmill_api_client::types::OpenFlowWPath {
+                open_flow: flow,
+                path: "g/all/flow_lock_all".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let mut str = listen_for_completed_jobs(&db).await;
+    let listen_first_job = str.next();
+    in_test_worker(&db, listen_first_job, port).await;
+
+    client
+        .get_flow_by_path("test-workspace", "g/all/flow_lock_all")
+        .await
+        .unwrap()
+        .into_inner()
+        .subtype_0
+        .value
+        .modules
+        .into_iter()
+        .for_each(|m| {
+            assert!(matches!(
+                m.value,
+                windmill_api_client::types::FlowModuleValue::Rawscript {
+                    language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
+                    lock: Some(ref lock),
+                    ..
+                } if lock == "")
+                || matches!(
+                m.value,
+                windmill_api_client::types::FlowModuleValue::Rawscript {
+                    language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
+                    lock: Some(ref lock),
+                    ..
+                } if lock.len() > 0)
+            );
+        });
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_rust_client(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    )
+    .list_workspaces()
+    .await
+    .unwrap();
+}

backend/windmill-api/Cargo.toml

@@ -12,6 +12,9 @@ path = "src/lib.rs"
 name = "windmill_api"
 path = "src/main.rs"
 
+[features]
+enterprise = ["windmill-queue/enterprise"]
+
 [dependencies]
 windmill-queue.workspace = true
 windmill-common = { workspace = true, features = [
@@ -64,3 +67,8 @@ tokio-util.workspace = true
 tokio-tar.workspace = true
 hmac.workspace = true
 cookie.workspace = true
+sha2.workspace = true
+urlencoding.workspace = true
+async-stripe.workspace = true
+lazy_static.workspace = true
+prometheus.workspace = true

backend/windmill-api/build_openapi.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+npx @redocly/openapi-cli@latest bundle openapi.yaml > openapi-bundled.yaml
+
+
+npx @redocly/openapi-cli@latest bundle openapi-bundled.yaml --ext yaml -d > openapi-deref.yaml
+
+rm openapi-bundled.yaml

backend/windmill-api/src/apps.rs

@@ -0,0 +1,748 @@
+use std::collections::HashMap;
+
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+use crate::{
+    db::{UserDB, DB},
+    jobs::script_path_to_payload,
+    users::{require_owner_of_path, Authed, OptAuthed},
+    variables::build_crypt,
+    webhook_util::{WebhookMessage, WebhookUtil},
+};
+use axum::{
+    extract::{Extension, Json, Path, Query},
+    routing::{delete, get, post},
+    Router,
+};
+use hyper::StatusCode;
+use magic_crypt::MagicCryptTrait;
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Map, Value};
+use sha2::{Digest, Sha256};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::{types::Uuid, FromRow};
+use std::str;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    apps::ListAppQuery,
+    error::{to_anyhow, Error, JsonResult, Result},
+    users::username_to_permissioned_as,
+    utils::{
+        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
+    },
+};
+use windmill_queue::{push, JobPayload, RawCode};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_apps))
+        .route("/get/p/*path", get(get_app))
+        .route("/secret_of/*path", get(get_secret_id))
+        .route("/get/v/*id", get(get_app_by_id))
+        .route("/exists/*path", get(exists_app))
+        .route("/update/*path", post(update_app))
+        .route("/delete/*path", delete(delete_app))
+        .route("/create", post(create_app))
+}
+
+pub fn unauthed_service() -> Router {
+    Router::new()
+        .route("/execute_component/*path", post(execute_component))
+        .route("/public_app/:secret", get(get_public_app_by_secret))
+}
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/hub/list", get(list_hub_apps))
+        .route("/hub/get/:id", get(get_hub_app_by_id))
+}
+
+#[derive(FromRow, Deserialize, Serialize)]
+pub struct ListableApp {
+    pub id: i64,
+    pub workspace_id: String,
+    pub path: String,
+    pub summary: String,
+    pub version: i64,
+    pub extra_perms: serde_json::Value,
+    pub execution_mode: String,
+    pub starred: bool,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+}
+
+#[derive(FromRow, Serialize, Deserialize)]
+pub struct AppVersion {
+    pub id: i64,
+    pub app_id: Uuid,
+    pub value: serde_json::Value,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct AppWithLastVersion {
+    pub id: i64,
+    pub path: String,
+    pub summary: String,
+    pub policy: serde_json::Value,
+    pub versions: Vec<i64>,
+    pub value: serde_json::Value,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+}
+
+pub type StaticFields = Map<String, Value>;
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[serde(rename_all = "lowercase")]
+pub enum ExecutionMode {
+    Anonymous,
+    Publisher,
+    Viewer,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Policy {
+    pub on_behalf_of: Option<String>,
+    pub on_behalf_of_email: Option<String>,
+    //paths:
+    // - script/<path>
+    // - flow/<path>
+    // - rawscript/<sha256>
+    pub triggerables: HashMap<String, StaticFields>,
+    pub execution_mode: ExecutionMode,
+}
+
+#[derive(Deserialize)]
+pub struct CreateApp {
+    pub path: String,
+    pub summary: String,
+    pub value: serde_json::Value,
+    pub policy: Policy,
+}
+
+#[derive(Deserialize)]
+pub struct EditApp {
+    pub path: Option<String>,
+    pub summary: Option<String>,
+    pub value: Option<serde_json::Value>,
+    pub policy: Option<Policy>,
+}
+
+async fn list_apps(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(lq): Query<ListAppQuery>,
+) -> JsonResult<Vec<ListableApp>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut sqlb = SqlBuilder::select_from("app")
+        .fields(&[
+            "app.id",
+            "app.workspace_id",
+            "app.path",
+            "app.summary",
+            "app.versions[array_upper(app.versions, 1)] as version",
+            "app.policy->>'execution_mode' as execution_mode",
+            "app_version.created_at as edited_at",
+            "app.extra_perms",
+            "favorite.path IS NOT NULL as starred",
+        ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .left()
+        .join("app_version")
+        .on(
+            "app_version.id = versions[array_upper(versions, 1)]"
+        )
+        .order_desc("favorite.path IS NOT NULL")
+        .order_by("app_version.created_at", true)
+        .and_where("app.workspace_id = ? OR app.workspace_id = 'starter'".bind(&w_id))
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
+
+    let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = sqlx::query_as::<_, ListableApp>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn get_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<AppWithLastVersion> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_as!(
+        AppWithLastVersion,
+        "SELECT app.id, app.path, app.summary, app.versions, app.policy,
+        app.extra_perms, app_version.value, 
+        app_version.created_at, app_version.created_by from app, app_version 
+        WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    Ok(Json(app))
+}
+
+async fn get_app_by_id(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, id)): Path<(String, i64)>,
+) -> JsonResult<AppWithLastVersion> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_as!(
+        AppWithLastVersion,
+        "SELECT app.id, app.path, app.summary, app.versions, app.policy,
+        app.extra_perms, app_version.value, 
+        app_version.created_at, app_version.created_by from app, app_version 
+        WHERE app_version.id = $1 AND app.id = app_version.app_id AND app.workspace_id = $2",
+        id,
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", id.to_string())?;
+    Ok(Json(app))
+}
+
+async fn get_public_app_by_secret(
+    Extension(db): Extension<DB>,
+    Path((w_id, secret)): Path<(String, String)>,
+) -> JsonResult<AppWithLastVersion> {
+    let mut tx = db.begin().await?;
+
+    let mc = build_crypt(&mut tx, &w_id).await?;
+
+    let decrypted = mc
+        .decrypt_bytes_to_bytes(&(hex::decode(secret)?))
+        .map_err(|e| Error::InternalErr(e.to_string()))?;
+    let bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?;
+
+    let id: i64 = bytes.parse().map_err(to_anyhow)?;
+
+    let app_o = sqlx::query_as!(
+        AppWithLastVersion,
+        "SELECT app.id, app.path, app.summary, app.versions, app.policy,
+        app.extra_perms, app_version.value, 
+        app_version.created_at, app_version.created_by from app, app_version 
+        WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
+        id,
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", id.to_string())?;
+
+    let policy = serde_json::from_value::<Policy>(app.policy.clone()).map_err(to_anyhow)?;
+
+    if !matches!(policy.execution_mode, ExecutionMode::Anonymous) {
+        return Err(Error::NotAuthorized(
+            "App visibility does not allow public access".to_string(),
+        ));
+    }
+
+    Ok(Json(app))
+}
+
+async fn get_secret_id(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let id_o = sqlx::query_scalar!(
+        "SELECT app.id FROM app
+        WHERE app.path = $1 AND app.workspace_id = $2",
+        path,
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    let id = not_found_if_none(id_o, "App", path.to_string())?;
+
+    let mc = build_crypt(&mut tx, &w_id).await?;
+
+    let hx = hex::encode(mc.encrypt_str_to_bytes(id.to_string()));
+
+    tx.commit().await?;
+
+    Ok(hx)
+}
+
+async fn create_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path(w_id): Path<String>,
+    Json(app): Json<CreateApp>,
+) -> Result<(StatusCode, String)> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let id = sqlx::query_scalar!(
+        "INSERT INTO app
+            (workspace_id, path, summary, policy, versions)
+            VALUES ($1, $2, $3, $4, '{}') RETURNING id",
+        w_id,
+        app.path,
+        app.summary,
+        json!(app.policy),
+    )
+    .fetch_one(&mut tx)
+    .await?;
+
+    let v_id = sqlx::query_scalar!(
+        "INSERT INTO app_version
+            (app_id, value, created_by)
+            VALUES ($1, $2, $3) RETURNING id",
+        id,
+        app.value,
+        authed.username,
+    )
+    .fetch_one(&mut tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE app SET versions = array_append(versions, $1) WHERE id = $2",
+        v_id,
+        id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&app.path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    webhook.send_message(WebhookMessage::CreateApp {
+        workspace: w_id.clone(),
+        path: app.path.clone(),
+    });
+
+    Ok((StatusCode::CREATED, app.path))
+}
+
+async fn list_hub_apps(
+    Authed { email, .. }: Authed,
+    Extension(http_client): Extension<Client>,
+) -> JsonResult<serde_json::Value> {
+    let flows = list_elems_from_hub(
+        http_client,
+        "https://hub.windmill.dev/searchUiData?approved=true",
+        &email,
+    )
+    .await?;
+    Ok(Json(flows))
+}
+
+pub async fn get_hub_app_by_id(
+    Authed { email, .. }: Authed,
+    Path(id): Path<i32>,
+    Extension(http_client): Extension<Client>,
+) -> JsonResult<serde_json::Value> {
+    let value = http_get_from_hub(
+        http_client,
+        &format!("https://hub.windmill.dev/apps/{id}/json"),
+        &email,
+        false,
+    )
+    .await?
+    .json()
+    .await
+    .map_err(to_anyhow)?;
+    Ok(Json(value))
+}
+
+async fn delete_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM app WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook
+        .send_message(WebhookMessage::DeleteApp { workspace: w_id.clone(), path: path.to_owned() });
+
+    Ok(format!("app {} deleted", path))
+}
+
+async fn update_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(ns): Json<EditApp>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let path = path.to_path();
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() {
+        let mut sqlb = SqlBuilder::update_table("app");
+        sqlb.and_where_eq("path", "?".bind(&path));
+        sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+        if let Some(npath) = &ns.path {
+            if npath != path {
+                if !authed.is_admin {
+                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
+                        .await?;
+                }
+            }
+            sqlb.set_str("path", npath);
+        }
+
+        if let Some(nsummary) = &ns.summary {
+            sqlb.set_str("summary", nsummary);
+        }
+
+        if let Some(npolicy) = ns.policy {
+            sqlb.set(
+                "policy",
+                &format!(
+                    "'{}'",
+                    serde_json::to_string(&json!(npolicy)).map_err(|e| {
+                        Error::BadRequest(format!("failed to serialize policy: {}", e))
+                    })?
+                ),
+            );
+        }
+
+        sqlb.returning("path");
+
+        let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+        not_found_if_none(npath_o, "App", path)?
+    } else {
+        "".to_string()
+    };
+    if let Some(nvalue) = &ns.value {
+        let app_id = sqlx::query_scalar!(
+            "SELECT id FROM app WHERE path = $1 AND workspace_id = $2",
+            path,
+            w_id
+        )
+        .fetch_one(&mut tx)
+        .await?;
+
+        let v_id = sqlx::query_scalar!(
+            "INSERT INTO app_version
+                (app_id, value, created_by)
+                VALUES ($1, $2, $3) RETURNING id",
+            app_id,
+            nvalue,
+            authed.username,
+        )
+        .fetch_one(&mut tx)
+        .await?;
+
+        sqlx::query!(
+            "UPDATE app SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3",
+            v_id,
+            path,
+            w_id
+        )
+        .execute(&mut tx)
+        .await?;
+    }
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&npath),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(WebhookMessage::UpdateApp {
+        workspace: w_id.clone(),
+        old_path: path.to_owned(),
+        new_path: npath.clone(),
+    });
+
+    Ok(format!("app {} updated (npath: {:?})", path, npath))
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct ExecuteApp {
+    pub args: Map<String, serde_json::Value>,
+    // - script: script/<path>
+    // - flow: flow/<path>
+    pub path: Option<String>,
+    pub raw_code: Option<RawCode>,
+    // if set, the app is executed as viewer with the given static fields
+    pub force_viewer_static_fields: Option<StaticFields>,
+}
+
+fn digest(code: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(code);
+    let result = hasher.finalize();
+    format!("rawscript/{:x}", result)
+}
+
+async fn execute_component(
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(payload): Json<ExecuteApp>,
+) -> Result<String> {
+    match (payload.path.is_some(), payload.raw_code.is_some()) {
+        (false, false) => {
+            return Err(Error::BadRequest(
+                "path or raw_code is required".to_string(),
+            ))
+        }
+        (true, true) => {
+            return Err(Error::BadRequest(
+                "path and raw_code cannot be set at the same time".to_string(),
+            ))
+        }
+        _ => {}
+    };
+
+    let path = path.to_path();
+    let mut tx = db.begin().await?;
+
+    let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
+        let mut hm = HashMap::new();
+        if let Some(path) = payload.path.clone() {
+            hm.insert(path, static_fields);
+        } else {
+            hm.insert(
+                digest(payload.raw_code.clone().unwrap().content.as_str()),
+                static_fields,
+            );
+        }
+        Policy {
+            execution_mode: ExecutionMode::Viewer,
+            triggerables: hm,
+            on_behalf_of: None,
+            on_behalf_of_email: None,
+        }
+    } else {
+        let policy_o = sqlx::query_scalar!(
+            "SELECT policy from app WHERE path = $1 AND workspace_id = $2",
+            path,
+            &w_id
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        let policy = not_found_if_none(policy_o, "App", path)?;
+
+        serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
+    };
+
+    let (username, permissioned_as, email) = match policy.execution_mode {
+        ExecutionMode::Anonymous => {
+            let username = opt_authed
+                .map(|a| a.username)
+                .unwrap_or_else(|| "anonymous".to_string());
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
+        }
+        ExecutionMode::Publisher => {
+            let username = opt_authed.map(|a| a.username).ok_or_else(|| {
+                Error::BadRequest("publisher execution mode requires authentication".to_string())
+            })?;
+            let (permissioned_as, email) = get_on_behalf_of(&policy)?;
+            (username, permissioned_as, email)
+        }
+        ExecutionMode::Viewer => {
+            let (username, email) = opt_authed.map(|a| (a.username, a.email)).ok_or_else(|| {
+                Error::BadRequest("Required to be authed in viewer mode".to_string())
+            })?;
+            (
+                username.clone(),
+                username_to_permissioned_as(&username),
+                email,
+            )
+        }
+    };
+
+    let (job_payload, args) = match &payload {
+        ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
+            let content = &raw_code.content;
+            let payload = JobPayload::Code(raw_code.clone());
+            let path = digest(content);
+            let args = build_args(policy, path, args)?;
+            (payload, args)
+        }
+        ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
+            let payload = if path.starts_with("script/") {
+                script_path_to_payload(path.strip_prefix("script/").unwrap(), &mut tx, &w_id)
+                    .await?
+            } else if path.starts_with("flow/") {
+                JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
+            } else {
+                return Err(Error::BadRequest(format!(
+                    "path must start with script/ or flow/ (got {})",
+                    path
+                )));
+            };
+            let args = build_args(policy, path.to_string(), args)?;
+            (payload, args)
+        }
+        _ => unreachable!(),
+    };
+
+    let (uuid, tx) = push(
+        tx,
+        &w_id,
+        job_payload,
+        args,
+        &username,
+        &email,
+        permissioned_as,
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+    )
+    .await?;
+
+    tx.commit().await?;
+    Ok(uuid.to_string())
+}
+
+fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
+    let permissioned_as = policy
+        .on_behalf_of
+        .as_ref()
+        .ok_or_else(|| {
+            Error::BadRequest(
+                "on_behalf_of is missing in the app policy and is required for anonymous execution"
+                    .to_string(),
+            )
+        })?
+        .to_string();
+    let email = policy
+        .on_behalf_of_email
+        .as_ref()
+        .ok_or_else(|| {
+            Error::BadRequest(
+                "on_behalf_of_email is missing in the app policy and is required for anonymous execution"
+                    .to_string(),
+            )
+        })?
+        .to_string();
+    Ok((permissioned_as, email))
+}
+
+async fn exists_app(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
+
+fn build_args(
+    policy: Policy,
+    path: String,
+    args: &Map<String, Value>,
+) -> Result<Map<String, Value>> {
+    let static_args = policy
+        .triggerables
+        .get(&path)
+        .map(|x| x.clone())
+        .or_else(|| {
+            if matches!(policy.execution_mode, ExecutionMode::Viewer) {
+                Some(Map::new())
+            } else {
+                None
+            }
+        })
+        .ok_or_else(|| {
+            Error::BadRequest(format!("path {} is not allowed in the app policy", path))
+        })?;
+    let mut args = args.clone();
+    for (k, v) in static_args {
+        args.insert(k.to_string(), v.to_owned());
+    }
+    Ok(args)
+}

backend/windmill-api/src/db.rs

@@ -72,6 +72,34 @@ impl UserDB {
         )
         .fetch_optional(&mut tx)
         .await?;
+
+        let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
+            &authed.folders.clone().into_iter().partition(|x| x.1);
+
+        let mut folders_read = folders_read.clone();
+        folders_read.extend(folders_write.clone());
+        sqlx::query!(
+            "SELECT set_config('session.folders_read', $1, true)",
+            folders_read
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.folders_write', $1, true)",
+            folders_write
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
         Ok(tx)
     }
 }

backend/windmill-api/src/favorite.rs

@@ -0,0 +1,75 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{db::DB, users::Authed};
+use axum::{
+    extract::{Extension, Path},
+    routing::post,
+    Json, Router,
+};
+use windmill_common::error::Result;
+
+use serde::{Deserialize, Serialize};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/star", post(star))
+        .route("/unstar", post(unstar))
+}
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "FAVORITE_KIND", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum FavoriteKind {
+    Script,
+    Flow,
+    App,
+}
+#[derive(Deserialize)]
+pub struct Favorite {
+    pub favorite_kind: FavoriteKind,
+    pub path: String,
+}
+
+async fn star(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(Favorite { favorite_kind, path }): Json<Favorite>,
+) -> Result<String> {
+    sqlx::query!(
+        "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
+        &w_id,
+        authed.username,
+        path,
+        favorite_kind: FavoriteKind,
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Starred {}", path))
+}
+
+async fn unstar(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(Favorite { favorite_kind, path }): Json<Favorite>,
+) -> Result<String> {
+    sqlx::query!(
+        "DELETE FROM favorite WHERE workspace_id = $1 AND usr = $2 AND path = $3 AND favorite_kind = $4",
+        &w_id,
+        authed.username,
+        path,
+        favorite_kind: FavoriteKind,
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Unstarred {}", path))
+}

backend/windmill-api/src/flows.rs

@@ -6,11 +6,12 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use hyper::StatusCode;
 use reqwest::Client;
 use sql_builder::prelude::*;
 
 use axum::{
-    extract::{Extension, Host, Path, Query},
+    extract::{Extension, Path, Query},
     routing::{get, post},
     Json, Router,
 };
@@ -19,15 +20,19 @@ use sqlx::{Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, to_anyhow, Error, JsonResult, Result},
-    flows::{Flow, ListFlowQuery, NewFlow},
+    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
+    schedule::Schedule,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
+use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
 
 use crate::{
     db::{UserDB, DB},
-    users::Authed,
+    schedule::clear_schedule,
+    users::{require_owner_of_path, Authed},
+    webhook_util::{WebhookMessage, WebhookUtil},
 };
 
 pub fn workspaced_service() -> Router {
@@ -38,6 +43,7 @@ pub fn workspaced_service() -> Router {
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
         .route("/exists/*path", get(exists_flow_by_path))
+        .route("/list_paths", get(list_paths))
 }
 
 pub fn global_service() -> Router {
@@ -52,24 +58,30 @@ async fn list_flows(
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListFlowQuery>,
-) -> JsonResult<Vec<Flow>> {
+) -> JsonResult<Vec<ListableFlow>> {
     let (per_page, offset) = paginate(pagination);
 
     let mut sqlb = SqlBuilder::select_from("flow as o")
         .fields(&[
-            "workspace_id",
-            "path",
+            "o.workspace_id",
+            "o.path",
             "summary",
             "description",
-            "'{}'::jsonb as value",
             "edited_by",
             "edited_at",
             "archived",
-            "null schema",
             "extra_perms",
+            "favorite.path IS NOT NULL as starred",
         ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
-        .and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
+        .and_where("o.workspace_id = ? OR o.workspace_id = 'starter'".bind(&w_id))
         .offset(offset)
         .limit(per_page)
         .clone();
@@ -86,42 +98,59 @@ async fn list_flows(
     if let Some(cb) = &lq.edited_by {
         sqlb.and_where_eq("edited_by", "?".bind(cb));
     }
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
 
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
-    let rows = sqlx::query_as::<_, Flow>(&sql).fetch_all(&mut tx).await?;
+    let rows = sqlx::query_as::<_, ListableFlow>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
     tx.commit().await?;
     Ok(Json(rows))
 }
 
 async fn list_hub_flows(
-    Authed { email, username, .. }: Authed,
+    Authed { email, .. }: Authed,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
         http_client,
         "https://hub.windmill.dev/searchFlowData?approved=true",
-        email,
-        username,
-        host,
+        &email,
     )
     .await?;
     Ok(Json(flows))
 }
 
+async fn list_paths(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+) -> JsonResult<Vec<String>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let flows = sqlx::query_scalar!(
+        "SELECT distinct(path) FROM flow WHERE  workspace_id = $1",
+        w_id
+    )
+    .fetch_all(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    Ok(Json(flows))
+}
+
 pub async fn get_hub_flow_by_id(
-    Authed { email, username, .. }: Authed,
+    Authed { email, .. }: Authed,
     Path(id): Path<i32>,
     Extension(http_client): Extension<Client>,
-    Host(host): Host,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
         http_client,
         &format!("https://hub.windmill.dev/flows/{id}/json"),
-        email,
-        username,
-        host,
+        &email,
         false,
     )
     .await?
@@ -131,20 +160,41 @@ pub async fn get_hub_flow_by_id(
     Ok(Json(value))
 }
 
+async fn check_path_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(Error::BadRequest(format!("Flow {} already exists", path)));
+    }
+    return Ok(());
+}
+
 async fn create_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
-) -> Result<String> {
+) -> Result<(StatusCode, String)> {
     // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx = user_db.clone().begin(&authed).await?;
 
+    check_path_conflict(&mut tx, &w_id, &nf.path).await?;
     check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
 
     sqlx::query!(
         "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json)",
+         schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
         w_id,
         nf.path,
         nf.summary,
@@ -173,7 +223,40 @@ async fn create_flow(
     .await?;
 
     tx.commit().await?;
-    Ok(nf.path.to_string())
+    webhook.send_message(WebhookMessage::CreateFlow {
+        workspace: w_id.clone(),
+        path: nf.path.clone(),
+    });
+
+    let tx = user_db.begin(&authed).await?;
+    let (dependency_job_uuid, mut tx) = push(
+        tx,
+        &w_id,
+        JobPayload::FlowDependencies { path: nf.path.clone() },
+        serde_json::Map::new(),
+        &authed.username,
+        &authed.email,
+        windmill_common::users::username_to_permissioned_as(&authed.username),
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+    )
+    .await?;
+    sqlx::query!(
+        "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
+        dependency_job_uuid,
+        nf.path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    Ok((StatusCode::CREATED, nf.path.to_string()))
 }
 
 async fn check_schedule_conflict<'c>(
@@ -202,30 +285,78 @@ async fn check_schedule_conflict<'c>(
 async fn update_flow(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    webhook: WebhookUtil,
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Json(nf): Json<NewFlow>,
 ) -> Result<String> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx = user_db.clone().begin(&authed).await?;
 
     let flow_path = flow_path.to_path();
     check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
 
     let schema = nf.schema.map(|x| x.0);
-    let flow = sqlx::query_scalar!(
+    let old_dep_job = sqlx::query_scalar!(
+        "SELECT dependency_job FROM flow WHERE path = $1 AND workspace_id = $2",
+        flow_path,
+        w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
+    sqlx::query!(
         "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6 WHERE path = $7 AND workspace_id = $8 RETURNING path",
+         edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
         nf.path,
         nf.summary,
         nf.description,
         nf.value,
         &authed.username,
-        schema,
+        schema.and_then(|x| serde_json::to_string(&x).ok()),
         flow_path,
         w_id,
     )
-    .fetch_optional(&mut tx)
+    .execute(&mut tx)
     .await?;
-    not_found_if_none(flow, "Flow", flow_path)?;
+
+    if nf.path != flow_path {
+        check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
+
+        if !authed.is_admin {
+            require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
+        }
+
+        let mut schedulables = sqlx::query_as!(
+            Schedule,
+                "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+                nf.path,
+                flow_path,
+                w_id,
+            )
+            .fetch_all(&mut tx)
+            .await?;
+
+        let schedule = sqlx::query_as!(Schedule,
+            "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+            nf.path,
+            flow_path,
+            w_id,
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        if let Some(schedule) = schedule {
+            schedulables.push(schedule);
+        }
+
+        for schedule in schedulables {
+            clear_schedule(&mut tx, flow_path, true).await?;
+
+            if schedule.enabled {
+                tx = push_scheduled_job(tx, schedule).await?;
+            }
+        }
+    }
 
     audit_log(
         &mut tx,
@@ -243,6 +374,47 @@ async fn update_flow(
     )
     .await?;
 
+    tx.commit().await?;
+    webhook.send_message(WebhookMessage::UpdateFlow {
+        workspace: w_id.clone(),
+        old_path: flow_path.to_owned(),
+        new_path: nf.path.clone(),
+    });
+
+    let tx = user_db.begin(&authed).await?;
+    let (dependency_job_uuid, mut tx) = push(
+        tx,
+        &w_id,
+        JobPayload::FlowDependencies { path: nf.path.clone() },
+        serde_json::Map::new(),
+        &authed.username,
+        &authed.email,
+        windmill_common::users::username_to_permissioned_as(&authed.username),
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+    )
+    .await?;
+    sqlx::query!(
+        "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
+        dependency_job_uuid,
+        nf.path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    if let Some(old_dep_job) = old_dep_job {
+        sqlx::query!(
+            "UPDATE queue SET canceled = true WHERE id = $1",
+            old_dep_job
+        )
+        .execute(&mut tx)
+        .await?;
+    }
     tx.commit().await?;
     Ok(nf.path.to_string())
 }
@@ -290,6 +462,7 @@ async fn exists_flow_by_path(
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
     let path = path.to_path();
@@ -314,6 +487,10 @@ async fn archive_flow_by_path(
     )
     .await?;
     tx.commit().await?;
+    webhook.send_message(WebhookMessage::ArchiveFlow {
+        workspace: w_id.clone(),
+        path: path.to_owned(),
+    });
 
     Ok(format!("Flow {path} archived"))
 }
@@ -347,6 +524,7 @@ mod tests {
                             InputTransform::Static { value: serde_json::json!("test2") },
                         )]
                         .into(),
+                        hash: None,
                     },
                     stop_after_if: None,
                     summary: None,
@@ -362,6 +540,7 @@ mod tests {
                         content: "test".to_string(),
                         language: scripts::ScriptLang::Deno,
                         path: None,
+                        lock: None,
                     },
                     stop_after_if: Some(StopAfterIf {
                         expr: "foo = 'bar'".to_string(),
@@ -397,6 +576,7 @@ mod tests {
                 value: FlowModuleValue::Script {
                     path: "test".to_string(),
                     input_transforms: HashMap::new(),
+                    hash: None,
                 },
                 stop_after_if: Some(StopAfterIf {
                     expr: "previous.isEmpty()".to_string(),
@@ -434,6 +614,7 @@ mod tests {
                 "input_transforms": {},
                 "type": "rawscript",
                 "content": "test",
+                "lock": null,
                 "path": null,
                 "language": "deno"
               },

backend/windmill-api/src/folders.rs

@@ -0,0 +1,570 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    db::{UserDB, DB},
+    users::Authed,
+    webhook_util::{WebhookMessage, WebhookUtil},
+};
+use axum::{
+    extract::{Extension, Path, Query},
+    routing::{delete, get, post},
+    Json, Router,
+};
+use itertools::Itertools;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    error::{self, Error, JsonResult, Result},
+    users::username_to_permissioned_as,
+    utils::{not_found_if_none, paginate, Pagination},
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::{query_scalar, FromRow, Postgres, Transaction};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_folders))
+        .route("/listnames", get(list_foldernames))
+        .route("/create", post(create_folder))
+        .route("/get/:name", get(get_folder))
+        .route("/update/:name", post(update_folder))
+        .route("/getusage/:name", get(get_folder_usage))
+        .route("/delete/:name", delete(delete_folder))
+        .route("/addowner/:name", post(add_owner))
+        .route("/removeowner/:name", post(remove_owner))
+        .route("/is_owner/*path", get(is_owner))
+}
+
+#[derive(FromRow, Serialize, Deserialize, Clone)]
+pub struct Folder {
+    pub workspace_id: String,
+    pub name: String,
+    pub display_name: String,
+    pub owners: Vec<String>,
+    pub extra_perms: serde_json::Value,
+}
+
+#[derive(Deserialize)]
+pub struct NewFolder {
+    pub name: String,
+    pub display_name: Option<String>,
+    pub owners: Option<Vec<String>>,
+    pub extra_perms: Option<serde_json::Value>,
+}
+
+#[derive(Deserialize)]
+pub struct UpdateFolder {
+    pub display_name: Option<String>,
+    pub owners: Option<Vec<String>>,
+    pub extra_perms: Option<serde_json::Value>,
+}
+
+#[derive(Deserialize)]
+pub struct Owner {
+    pub owner: String,
+}
+
+async fn list_folders(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+) -> JsonResult<Vec<Folder>> {
+    let (per_page, offset) = paginate(pagination);
+    let mut tx = user_db.begin(&authed).await?;
+
+    let rows = sqlx::query_as!(
+        Folder,
+        "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        w_id,
+        per_page as i64,
+        offset as i64
+    )
+    .fetch_all(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+async fn list_foldernames(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+) -> JsonResult<Vec<String>> {
+    let (per_page, offset) = paginate(pagination);
+    let mut tx = user_db.begin(&authed).await?;
+
+    let rows = sqlx::query_scalar!(
+        "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
+        w_id,
+        per_page as i64,
+        offset as i64
+    )
+    .fetch_all(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn check_name_conflict<'c>(
+    tx: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<()> {
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)",
+        name,
+        w_id
+    )
+    .fetch_one(tx)
+    .await?
+    .unwrap_or(false);
+    if exists {
+        return Err(windmill_common::error::Error::BadRequest(format!(
+            "Folder {} already exists",
+            name
+        )));
+    }
+    return Ok(());
+}
+
+async fn create_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path(w_id): Path<String>,
+    Json(ng): Json<NewFolder>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    check_name_conflict(&mut tx, &w_id, &ng.name).await?;
+    let owner = username_to_permissioned_as(&authed.username);
+    let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
+
+    if let Some(extra_perms) = ng.extra_perms.clone() {
+        for o in owners {
+            if !extra_perms
+                .get(&o)
+                .and_then(|x| x.as_bool())
+                .unwrap_or(false)
+            {
+                return Err(windmill_common::error::Error::BadRequest(format!(
+                    "Owner {} would not have permission to write to folder and that is an inconsistent state",
+                    o
+                )));
+            }
+        }
+    }
+
+    let extra_perms = ng.extra_perms.unwrap_or_else(|| {
+        let mut map = serde_json::Map::new();
+        for o in owners {
+            map.insert(o.clone(), serde_json::json!(true));
+        }
+        serde_json::Value::Object(map)
+    });
+
+    sqlx::query_as!(
+        Folder,
+        "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, $2, $3, $4, $5)",
+        w_id,
+        ng.name,
+        ng.display_name.unwrap_or(ng.name.clone()),
+        owners,
+        extra_perms,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&ng.name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(WebhookMessage::CreateFolder {
+        workspace: w_id.clone(),
+        name: ng.name.clone(),
+    });
+
+    Ok(format!("Created folder {}", ng.name))
+}
+
+pub async fn is_owner(
+    Authed { username, is_admin, groups, .. }: Authed,
+    Extension(db): Extension<DB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<bool> {
+    if is_admin {
+        Ok(Json(true))
+    } else {
+        Ok(Json(
+            require_is_owner(&name, &username, &groups, &w_id, &db)
+                .await
+                .is_ok(),
+        ))
+    }
+}
+
+pub async fn require_is_owner(
+    folder_name: &str,
+    username: &str,
+    groups: &Vec<String>,
+    w_id: &str,
+    db: &DB,
+) -> Result<()> {
+    let is_owner = query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
+            SELECT 1 FROM folder, unnest(folder.owners) as o
+            WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
+        username,
+        folder_name,
+        groups,
+        w_id,
+    ).fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !is_owner {
+        Err(Error::BadRequest(format!(
+            "{} is not an owner of {} and hence is not authorized to perform this operation",
+            username, folder_name
+        )))
+    } else {
+        Ok(())
+    }
+}
+
+async fn update_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(ng): Json<UpdateFolder>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let mut sqlb = SqlBuilder::update_table("folder");
+    sqlb.and_where_eq("name", "?".bind(&name));
+    sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+    if let Some(display_name) = ng.display_name {
+        sqlb.set("display_name", display_name);
+    }
+    if let Some(owners) = ng.owners {
+        sqlb.set_str("owners", format!("{{{}}}", owners.into_iter().join(",")));
+    }
+    if let Some(extra_perms) = ng.extra_perms {
+        sqlb.set_str("extra_perms", extra_perms.to_string());
+    }
+
+    sqlb.returning("*");
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let sql = sqlb
+        .sql()
+        .map_err(|e| error::Error::InternalErr(e.to_string()))?;
+    let nfolder = sqlx::query_as::<_, Folder>(&sql).fetch_one(&mut tx).await?;
+
+    if let Some(extra_perms) = nfolder.extra_perms.as_object() {
+        for o in nfolder.owners {
+            if !extra_perms
+                .get(&o)
+                .and_then(|x| x.as_bool())
+                .unwrap_or(false)
+            {
+                return Err(windmill_common::error::Error::BadRequest(format!(
+                    "Owner {} would not have permission to write to folder and that is an invalid state",
+                    o
+                )));
+            }
+        }
+    }
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(WebhookMessage::UpdateFolder {
+        workspace: w_id.clone(),
+        name: name.to_owned(),
+    });
+
+    Ok(format!("Updated folder {}", name))
+}
+
+pub async fn get_folderopt<'c>(
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    name: &str,
+) -> Result<Option<Folder>> {
+    let folderopt = sqlx::query_as!(
+        Folder,
+        "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE name = $1 AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?;
+    Ok(folderopt)
+}
+
+async fn get_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<Folder> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+
+    tx.commit().await?;
+    Ok(Json(folder))
+}
+
+#[derive(Serialize)]
+struct FolderUsage {
+    pub scripts: i64,
+    pub schedules: i64,
+    pub flows: i64,
+    pub apps: i64,
+    pub resources: i64,
+    pub variables: i64,
+}
+async fn get_folder_usage(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, name)): Path<(String, String)>,
+) -> JsonResult<FolderUsage> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    let scripts = sqlx::query_scalar!(
+        "SELECT count(path) FROM script WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let schedules = sqlx::query_scalar!(
+        "SELECT count(path) FROM schedule WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let flows = sqlx::query_scalar!(
+        "SELECT count(path) FROM flow WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let apps = sqlx::query_scalar!(
+        "SELECT count(path) FROM app WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let resources = sqlx::query_scalar!(
+        "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    let variables = sqlx::query_scalar!(
+        "SELECT count(path) FROM variable WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
+    tx.commit().await?;
+    Ok(Json(FolderUsage {
+        scripts,
+        flows,
+        schedules,
+        apps,
+        resources,
+        variables,
+    }))
+}
+
+async fn delete_folder(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path((w_id, name)): Path<(String, String)>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+
+    sqlx::query!(
+        "DELETE FROM folder WHERE name = $1 AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(&name.to_string()),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+
+    webhook
+        .send_message(WebhookMessage::DeleteFolder { workspace: w_id.clone(), name: name.clone() });
+
+    Ok(format!("delete folder at name {}", name))
+}
+
+async fn add_owner(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(Owner { owner }): Json<Owner>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
+
+    sqlx::query!(
+        "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
+        owner,
+        name,
+        &w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.add_owner",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        Some([("owner", owner.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    webhook
+        .send_message(WebhookMessage::UpdateFolder { workspace: w_id.clone(), name: name.clone() });
+
+    Ok(format!("Added {} to folder {}", owner, name))
+}
+
+pub async fn get_folders_for_user(
+    w_id: &str,
+    username: &str,
+    groups: &[String],
+    db: &DB,
+) -> Result<Vec<(String, bool)>> {
+    let mut perms = groups
+        .into_iter()
+        .map(|x| format!("g/{}", x))
+        .collect::<Vec<_>>();
+    perms.insert(0, format!("u/{}", username));
+    let folders = sqlx::query!(
+        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write  FROM folder
+        WHERE extra_perms ?| $1  AND workspace_id = $2",
+        &perms[..],
+        w_id,
+    )
+    .fetch_all(db)
+    .await?
+    .into_iter()
+    .map(|x| (x.name, x.write.unwrap_or(false)))
+    .collect();
+
+    Ok(folders)
+}
+
+async fn remove_owner(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    webhook: WebhookUtil,
+    Path((w_id, name)): Path<(String, String)>,
+    Json(Owner { owner }): Json<Owner>,
+) -> Result<String> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
+    if !authed.is_admin {
+        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
+    }
+
+    sqlx::query!(
+        "UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",
+        owner,
+        name,
+        &w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "folder.remove_owner",
+        ActionKind::Update,
+        &w_id,
+        Some(&name.to_string()),
+        Some([("owner", owner.as_str())].into()),
+    )
+    .await?;
+    tx.commit().await?;
+
+    webhook
+        .send_message(WebhookMessage::UpdateFolder { workspace: w_id.clone(), name: name.clone() });
+
+    Ok(format!("Removed {} to folder {}", owner, name))
+}