feat(frontend): add a dev docker-compose file + update readme

* display previous script inputs on script run page

* parallelise loading

* parallelise loading

* also working for flows

* separate endpoints for scripts and flows

* Splitpanes and Saved Inputs (UI)

* Saved inputs API endpoints

* Editable Input name

* Narrow width styling

* feat(frontend): Add a toggle to open the saved inputs (#1401)

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Move toggle

* update all

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
Co-authored-by: Faton Ramadani <faton.ramadani14@gmail.com>

.env

@@ -1,2 +1,7 @@
 DB_PASSWORD=changeme
-WM_BASE_URL=windmill.localhost
+
+# this is the url that your instance is publicly exposed to
+WM_BASE_URL=http://localhost
+
+# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
+CADDY_REVERSE_PROXY=http://localhost

.github/DockerfilePypiBuilder

@@ -0,0 +1,7 @@
+FROM nikolaik/python-nodejs:python3.11-nodejs19
+
+RUN python3 -m pip install pipx poetry
+RUN python3 -m pipx ensurepath
+ENV PATH="/root/.local/bin:${PATH}"
+ENV PATH="/usr/local/bin:${PATH}"
+RUN pipx install openapi-python-client==0.11.6 --include-deps

.github/change-versions.sh

@@ -4,7 +4,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
-sed -i -e "/^const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json

.github/pull_hub_items.sh

@@ -9,6 +9,6 @@ for item in ${RT[@]}; do
     jq -r '.resource_type.schema' <<< "$body" > ./tmp
     description=$(jq -r '.resource_type.description' <<< "$body")
     description=$(echo -E $description)
-    echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
+    echo "{\"workspace_id\": \"admins\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
     rm ./tmp
 done

.github/uffizzi/caddy/Caddyfile

@@ -0,0 +1,5 @@
+localhost {
+        bind 0.0.0.0
+        reverse_proxy /ws/* http://0.0.0.0:3001
+        reverse_proxy /* http://0.0.0.0:8000
+}

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,53 @@
+version: '3.7'
+
+x-uffizzi:
+  ingress:
+    service: windmill
+    port: 8000
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: windmill
+
+  windmill:
+    image: '${WINDMILL_IMAGE}'
+    ports:
+      - 8000:8000
+    entrypoint: ['/bin/sh', '-c']
+    command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
+    environment:
+      - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
+      - BASE_URL=${EXPECTED_URL}
+      - BASE_INTERNAL_URL=http://localhost:8000
+      - RUST_LOG=info
+      - NUM_WORKERS=3
+      - KEEP_JOB_DIR=false
+      - DENO_PATH=/usr/bin/deno
+      - PYTHON_PATH=/usr/local/bin/python3
+      - METRICS_ADDR=false
+      - OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
+    volumes:
+      - worker_dependency_cache:/tmp/windmill/cache
+    deploy:
+      resources:
+        limits:
+          memory: 250M
+
+  lsp:
+    image: '${LSP_IMAGE}'
+    ports:
+      - 3001:3001
+
+  # caddy:
+  #   image: caddy:2.5.2-alpine
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./.github/uffizzi/caddy:/etc/caddy
+  #   environment:
+  #     - BASE_URL=localhost
+
+volumes:
+  worker_dependency_cache:

.github/workflows/automerge-dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.3.4
+        uses: dependabot/fetch-metadata@v1.3.6
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Enable auto-merge for Dependabot PRs

.github/workflows/backend-test.yml

@@ -39,5 +39,5 @@ jobs:
             backend
             backend -> target
       - name: cargo test
-        timeout-minutes: 5
-        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        timeout-minutes: 10
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture

.github/workflows/change-versions.yml

@@ -10,6 +10,12 @@ jobs:
     container: node:18
     steps:
       - uses: actions/checkout@v3
+      - run: git config --system --add safe.directory /__w/windmill/windmill
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: update lockfile
+        run: cd backend && cargo generate-lockfile
       - uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/deploy_to_windmill.yml

@@ -1,20 +0,0 @@
-name: Deploy to windmill.dev
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "community/**"
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Deploy to windmill.dev
-        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
-        with:
-          dry_run: false
-          input_dir: community
-          windmill_workspace: starter
-          windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}

.github/workflows/docker-310.yml

@@ -0,0 +1,47 @@
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build and push windmill with python 3.10 and openbb
+on: workflow_dispatch
+
+concurrency:
+  group: ${{ github.ref }}-openbb
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+  
+jobs:
+  build_ee:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileOpenbb
+          build-args: |
+            features=enterprise
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:openbb
+          labels: |
+            org.opencontainers.image.licenses=Windmill-Enterprise-License

.github/workflows/docker-image-arm.yml.archived

@@ -0,0 +1,60 @@
+# env:
+#   REGISTRY: ghcr.io
+#   IMAGE_NAME: ${{ github.repository }}
+
+# name: Build and push arm docker image
+# on:
+#   push:
+#     branches: [main]
+#     tags: ["*"]
+
+# concurrency:
+#   group: ${{ github.ref }}-arm
+#   cancel-in-progress: true
+
+# permissions:
+#   contents: read
+#   id-token: write
+#   packages: write
+
+# jobs:
+#   publish_arm:
+#     runs-on: ubuntu-22.04
+#     steps:
+#       - uses: actions/checkout@v3
+#         with:
+#           fetch-depth: 0
+
+#       - uses: depot/setup-action@v1
+
+#       - name: Docker meta
+#         id: meta-slim-public
+#         uses: docker/metadata-action@v4
+#         with:
+#           images: |
+#             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+#           tags: |
+#             type=ref,event=branch
+#             type=ref,event=pr
+#             type=semver,pattern={{version}}
+#             type=semver,pattern={{major}}.{{minor}}
+
+#       - name: Login to registry
+#         uses: docker/login-action@v2
+#         with:
+#           registry: ${{ env.REGISTRY }}
+#           username: ${{ github.actor }}
+#           password: ${{ secrets.GITHUB_TOKEN }}
+
+#       - name: Build and push publicly
+#         uses: depot/build-push-action@v1
+#         with:
+#           context: .
+#           push: true
+#           platforms: linux/amd64,linux/arm64
+#           tags: |
+#             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+#             ${{ steps.meta-slim-public.outputs.tags }}
+#           labels: |
+#             ${{ steps.meta-slim-public.outputs.labels }}
+#             org.opencontainers.image.licenses=AGPLv3

.github/workflows/docker-image.yml

@@ -1,6 +1,7 @@
 env:
   LOCAL_REGISTRY: registry.wimill.xyz
   REGISTRY: ghcr.io
+  ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com
   IMAGE_NAME: ${{ github.repository }}
 
 name: Build and push docker image
@@ -8,52 +9,77 @@ on:
   push:
     branches: [main]
     tags: ["*"]
-  pull_request:
-    types: [opened, synchronize, reopened]
 
 concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+
 jobs:
   build:
-    runs-on: [self-hosted, new]
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Docker meta local
-        id: metalocal
+
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+      - uses: depot/setup-action@v1
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta-public
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-      - name: Build and push privately
-        uses: docker/build-push-action@v3
-        if: github.event_name == 'pull_request'
+      - name: Build and push publicly
+        uses: depot/build-push-action@v1
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ${{ steps.metalocal.outputs.tags }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.meta-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-public.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
 
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+  build_ee:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
 
       - name: Docker meta
-        if: github.event_name != 'pull_request'
-        id: meta
+        id: meta-ee-public
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -61,51 +87,175 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
 
       - name: Login to registry
-        if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push publicly
-        uses: docker/build-push-action@v3
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          push: true
+          build-args: |
+            features=enterprise
+            nsjail=true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+
+  # disabled until we make it 100% reliable and add more meaningful tests
+  # playwright:
+  #   runs-on: [self-hosted, new]
+  #   needs: [build]
+  #   services:
+  #     postgres:
+  #       image: postgres
+  #       env:
+  #         POSTGRES_DB: windmill
+  #         POSTGRES_USER: admin
+  #         POSTGRES_PASSWORD: changeme
+  #       ports:
+  #         - 5432:5432
+  #       options: >-
+  #         --health-cmd pg_isready
+  #         --health-interval 10s
+  #         --health-timeout 5s
+  #         --health-retries 5
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: "Docker"
+  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+  #       id: docker-container
+  #     - uses: actions/setup-node@v3
+  #       with:
+  #         node-version: 16
+  #     - name: "Playwright run"
+  #       timeout-minutes: 2
+  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
+  #     - name: "Clean up"
+  #       run: docker kill ${{ steps.docker-container.outputs.id }}
+  #       if: always()
+
+  deploy_s3:
+    needs: [build_ee]
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - uses: shrink/actions-docker-extract@v2
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/static_frontend/."
+
+      - uses: reggionick/s3-deploy@v3
+        with:
+          folder: ${{ steps.extract.outputs.destination }}
+          bucket: windmill-frontend
+          bucket-region: us-east-1
+  publish_privately_heavy:
+    needs: [build_ee]
+    runs-on: [self-hosted, new]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta-heavy
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push privately
+        uses: docker/build-push-action@v4
         if: github.event_name != 'pull_request'
         with:
           context: .
           push: true
+          file: ./docker/DockerfileHeavy
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.metalocal.outputs.tags }}
-            ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
-  playwright:
+            ${{ steps.meta-heavy.outputs.tags }}
+          labels: ${{ steps.meta-heavy.outputs.labels }}
+          cache-from:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-heavy:buildcache
+          cache-to:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-heavy:buildcache,mode=max
+
+  publish_privately_helm:
     runs-on: [self-hosted, new]
-    needs: [build]
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_USER: admin
-          POSTGRES_PASSWORD: changeme
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+    needs: [build_ee]
+    if: github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@v3
-      - name: "Docker"
-        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-        id: docker-container
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
-      - name: "Clean up"
-        run: docker kill ${{ steps.docker-container.outputs.id }}
-        if: always()
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Build and push privately
+        uses: docker/build-push-action@v4
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          file: ./docker/DockerfileHelm
+          tags: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
+          cache-from:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-helm:buildcache
+          cache-to:
+            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
+            }}-helm:buildcache,mode=max

.github/workflows/frontend-check.yml

@@ -0,0 +1,17 @@
+name: check frontend build
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+    paths:
+      - "frontend/**"
+jobs:
+  npm_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: "npm check"
+        timeout-minutes: 2
+        run: cd frontend && npm ci && npm run generate-backend-client && npm run check

.github/workflows/go_on_release.yml

@@ -13,13 +13,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
       - name: generate_go
         run: |
-          go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@latest
+          go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.11.0
           cd go-client
           rm .gitignore
           ./build.sh
+          go build
       - name: Pushes to another repository
         id: push_directory
         uses: cpina/github-action-push-to-another-repository@devel

.github/workflows/pull-hub.yml

@@ -1,20 +0,0 @@
-name: Pull Hub Items
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 */1 * *"
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  change_version:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Pull hub
-        run: ./.github/pull_hub_items.sh
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
-        with:
-          title: sync hub items with community
-          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -1,5 +1,6 @@
 env:
   REGISTRY: ghcr.io
+  ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com
   IMAGE_NAME: ${{ github.repository }}-lsp
 
 name: Publish python-client
@@ -11,7 +12,9 @@ on:
 
 jobs:
   publish_pypi:
-    runs-on: [self-hosted, new]
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/windmill-labs/python-client-builder
     steps:
       - uses: actions/checkout@v3
       - name: Upload python client
@@ -19,14 +22,15 @@ jobs:
           PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           cd python-client
-          export PATH=$PATH:/usr/local/bin
-          export PATH=$PATH:/root/.local/bin
           ./publish.sh
 
   publish_lsp:
     needs: [publish_pypi]
     runs-on: [self-hosted, new]
     steps:
+      - name: Sleep for 30 seconds waiting for pypi to update index
+        run: sleep 30s
+        shell: bash
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -37,7 +41,9 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -51,15 +57,24 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Login to ECR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
       - name: Build and push publicly
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: "{{defaultContext}}:lsp"
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
             ${{ steps.metalocal.outputs.tags }}
             ${{ steps.meta.outputs.tags }}
+            registry.uffizzi.com/windmill-lsp:60d
           labels: ${{ steps.metalocal.outputs.labels }}
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
           cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/uffizzi-build.yml.archived

@@ -0,0 +1,93 @@
+name: Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+    paths:
+      - "backend/**"
+      - ".github/uffizzi/**"
+      - ".github/workflows/**"
+  workflow_dispatch:
+      
+jobs:
+  build-windmill:
+    name: Build and Push `windmill`
+    runs-on: ubuntu-latest
+    if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: ./
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs:
+      - build-windmill
+    outputs:
+      compose-file-cache-key: ${{ steps.hash.outputs.hash }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
+          export WINDMILL_IMAGE
+          LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
+          export LSP_IMAGE
+          envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }}
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run: echo '${{ toJSON(github.event) }}' > event.json
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview.yml.archived

@@ -0,0 +1,115 @@
+name: Deploy Uffizzi Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            if (matchArtifact === undefined) {
+              throw TypeError('Build Artifact not found!');
+            }
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      - name: Predict Deployment URL
+        id: url
+        # Replace dots in the repo name with the plus sign
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
+
+      - name: Re-Render Compose File
+        run: |
+          OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
+          export OAUTH_JSON_BASE64
+          envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
+          # cat docker-compose.uffizzi.yml
+
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.uffizzi.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.uffizzi.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+
+  playwright:
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-uffizzi-preview
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: "Playwright run"
+        timeout-minutes: 2
+        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test

.gitignore

@@ -4,3 +4,4 @@ nohup.out
 local/
 frontend/src/routes/test.svelte
 CaddyfileRemoteMalo
+*.swp

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic"
+}

Caddyfile

@@ -1,5 +1,5 @@
 {$BASE_URL} {
         bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
-        reverse_proxy /* http://windmill:8000
-}
+        reverse_proxy /* http://windmill_server:8000
+}

Dockerfile

@@ -1,8 +1,10 @@
-FROM python:3.11.0-slim-buster as nsjail
+FROM debian:buster-slim as nsjail
 
 WORKDIR /nsjail
 
-RUN apt-get -y update \
+ARG nsjail=""
+
+RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
     bison=2:3.3.* \
     flex=2.6.* \
@@ -13,11 +15,12 @@ RUN apt-get -y update \
     libnl-route-3-dev=3.4.* \
     make=4.2.* \
     pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*
+    protobuf-compiler=3.6.*; fi
 
-RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
-    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
-RUN make
+
+RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
+RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
 FROM rust:slim-buster AS rust_base
 
@@ -29,7 +32,7 @@ RUN apt-get -y update \
 
 RUN rustup component add rustfmt
 
-RUN cargo install cargo-chef 
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef 
 
 WORKDIR /windmill
 
@@ -48,67 +51,90 @@ COPY frontend .
 RUN mkdir /backend
 COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
 COPY /openflow.openapi.yaml /openflow.openapi.yaml
+COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
+RUN cd /backend/windmill-api && . ./build_openapi.sh
+
 RUN npm run generate-backend-client
 ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
 RUN npm run check
 
 
+
 FROM rust_base AS planner
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
 
-RUN cargo chef prepare  --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json
 
 FROM rust_base AS builder
+ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
 
 COPY --from=frontend /frontend /frontend
+COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml
 COPY .git/ .git/
 
-RUN cargo build --release
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:buster-slim
+FROM python:3.11.3-slim-buster
+ARG TARGETPLATFORM
+
 ARG APP=/usr/src/app
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
-    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
-    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
-    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip \
+    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    curl -o rclone.zip "https://downloads.rclone.org/v1.60.1/rclone-v1.60.1-linux-$arch.zip"; \
+    unzip -p rclone.zip rclone-v1.60.1-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
+    chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone
+
+RUN set -eux; \
+    arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+    url=; \
+    case "$arch" in \
+    'amd64') \
+    targz='go1.19.3.linux-amd64.tar.gz'; \
+    ;; \
+    'arm64') \
+    targz='go1.19.3.linux-arm64.tar.gz'; \
+    ;; \
+    'armhf') \
+    targz='go1.19.3.linux-armv6l.tar.gz'; \
+    ;; \
+    *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
+    esac; \
+    wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz";
+
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
 ENV TZ=Etc/UTC
 
-ENV PYTHON_VERSION 3.10.4
-
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
-    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
-    && make -j 4 && make install
-
 RUN /usr/local/bin/python3 -m pip install pip-tools
-RUN /usr/local/bin/python3 -m pip install nltk
-RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
+COPY --from=frontend /frontend/build /static_frontend
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
+# docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
+COPY --from=lukechannings/deno:latest /usr/bin/deno /usr/bin/deno-arm
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
+
 RUN mkdir -p ${APP}
 
 WORKDIR ${APP}

LICENSE

@@ -8,5 +8,9 @@ or belonging to one of the below cases:
 
 The files under backend/ are AGPL Licensed.
 The files under frontend/ are AGPL Licensed.
-The files under python-client/ are Apache 2.0 Licensed.
-The files under community/ are Apache 2.0 Licensed.
+The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+
+The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
+
+All third party components incorporated into the Windmill Software are licensed under the
+original license provided by the owner of the applicable component.

README.md

@@ -5,7 +5,7 @@
     <em>.</em>
 </p>
 <p align=center>
-Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger flows and scripts as internal apps. Convert code to no-code modules and, if the auto-generated UI is not sufficient, use it solely as an highly scalable backend layer. Add automation to your product or build your own no-code tool and delegate the core layer to Windmill. The script languages supported are: Python, Typescript and Go.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
@@ -20,92 +20,112 @@ Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks an
 </a>
 </p>
 
----
-
-**Join the beta (personal workspaces are free forever)**:
-<https://app.windmill.dev>
-
-**Documentation**: <https://docs.windmill.dev>
-
-**Discord**: <https://discord.gg/V7PM2YHsPB>
-
-**Hub**: <https://hub.windmill.dev>
-
-**Contributor's guide**: <https://docs.windmill.dev/docs/contributors_guide>
-
-**Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
-
-**[Self-host instruction](#how-to-self-host)**
-
-You can show your support for the project by starring this repo.
-
----
-
-Windmill Labs offers commercial licenses and support to convert your existing
-automation and help you scale it in production. If interested, contact
-ruben@windmill.dev (founder of Windmill).
-
----
-
-# Windmill
-
 <p align="center">
-<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
+  <a href="https://app.windmill.dev">Try it</a> - <a href="https://docs.windmill.dev/docs/intro/">Docs</a> - <a href="https://discord.gg/V7PM2YHsPB">Discord</a> - <a href="https://hub.windmill.dev">Hub</a> - <a href="https://docs.windmill.dev/docs/misc/contributing">Contributor's guide</a>
 </p>
 
-![Windmill Screenshot](./imgs/windmill.png)
+# Windmill - Turn scripts into workflows and UIs that you can share and run at scale
 
-Windmill is <b>fully open-sourced</b>:
+Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
+dedicated instance and commercial support and licenses.
 
-- `community/`, `python-client/` and `deno-client/` are Apache 2.0
-- backend, frontend and everything else under AGPLv3.
+![Windmill Diagram](/imgs/stacks.svg)
 
-## What is the general idea behind Windmill
+https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2bd-3bacc1f0ec79.mp4
 
-1. Define a minimal and generic script in Python, Typescript or Go that solves a
-   specific task. Here sending an email with SMTP. The code can be defined in
-   the provided Web IDE or synchronized with your own github repo:
-   ![Step 1](./imgs/python-script.png)
+- [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
+  - [Main Concepts](#main-concepts)
+  - [Show me some actual script code](#show-me-some-actual-script-code)
+  - [CLI](#cli)
+    - [Running scripts locally](#running-scripts-locally)
+  - [Stack](#stack)
+  - [Security](#security)
+    - [Sandboxing](#sandboxing)
+    - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
+  - [Performance](#performance)
+  - [Architecture](#architecture)
+  - [How to self-host](#how-to-self-host)
+    - [Docker compose](#docker-compose)
+    - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
+    - [Postgres without superuser](#postgres-without-superuser)
+    - [Commercial license](#commercial-license)
+    - [OAuth for self-hosting](#oauth-for-self-hosting)
+    - [Resource types](#resource-types)
+  - [Environment Variables](#environment-variables)
+  - [Run a local dev setup](#run-a-local-dev-setup)
+    - [only Frontend](#only-frontend)
+    - [Backend + Frontend](#backend--frontend)
+  - [Contributors](#contributors)
+  - [Copyright](#copyright)
 
-2. Your scripts parameters are automatically parsed and generate a frontend. You
-   can narrow down the types during task definition to specify regex for string,
-   an enum or a specific format for objects. Each script correspond to an app by
-   itself: ![Step 2](./imgs/arguments.png)
+## Main Concepts
+
+1. Define a minimal and generic script in Python, Typescript, Go or Bash that
+   solves a specific task. Here sending an email with SMTP. The code can be
+   defined in the provided Web IDE or synchronized with your own github repo:
+   ![Step 1](./imgs/windmill-editor.png)
+
+2. Your scripts parameters are automatically parsed and generate a frontend.
+   ![Step 2](./imgs/windmill-run.png) ![Step 3](./imgs/windmill-result.png)
 
 3. Make it flow! You can chain your scripts or scripts made by the community
-   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
-   between Windmill and the hub to make it easy to build flows from a soon-to-be
-   exhaustive library of generic modules. In flows, one can pipe output to input
-   using "Dynamic" expressions that are just plain Javascript underneath. Flows
-   can contain for-loops, branching (coming soon). As such and coupled with
-   inputs being able to refer to any step's output, they are actual DAG rather
-   than just linear sequences. They are backed by an open JSON spec we call
-   [OpenFlow](https://docs.windmill.dev/docs/openflow)
+   shared on [WindmillHub](https://hub.windmill.dev).
+   ![Step 3](./imgs/windmill-flow.png)
 
-Both scripts and flows are not restricted to be triggered by the UI. They can be
-triggered by a schedule, watch for changes (using
-[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
-triggered through API with either an async or sync webhook. The latter kind of
-endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
-central place to host, build and run all of your integrations, automation and
-internal apps. We include credentials management and OAuth integration, groups
-and much more!
+4. Build complex UI on top of your scripts and flows.
+   ![Step 4](./imgs/windmill-builder.png)
 
-## Layout
+Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
+through webhooks.
 
-- `backend/`: The whole Rust backend
-- `frontend`: The whole Svelte frontend
-- `community/`: Scripts and resource types included in every workspace. It is
-  useful for Python scripts since the [WindmillHub](https://hub.windmill.dev)
-  only allow deno scripts and for sharing resource types that will be included
-  in every workspace.
-- `lsp/`: The lsp asssistant for the monaco editor
-- `nsjail/`: The nsjail configuration files for sandboxing of the scripts'
-  execution
-- `python-client/`: The wmill python client used within scripts to interact with
-  the windmill platform
-- `deno-client/`: The wmill deno client used within scripts to interact with the
-  windmill platform
+You can build your entire infra on top of Windmill!
+
+## Show me some actual script code
+
+```typescript
+import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts";
+//import any dependency  from npm
+
+import cowsay from "npm:cowsay@1.5.0";
+
+export async function main(
+  a: number,
+  // unions generate enums
+  b: "my" | "enum",
+  // default parameters prefill the field
+  d = "default arg",
+  // nested objects work c = { nested: "object" },
+  // permissioned and typed json
+  db: wmill.Resource<"postgresql">
+) {
+  const email = Deno.env.get("WM_EMAIL");
+  // variables are permissioned and by path
+  let variable = await wmill.getVariable("f/company-folder/my_secret");
+  const lastTimeRun = await wmill.getState();
+  // logs are printed and always inspectable
+  console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }));
+  await wmill.setState(Date.now());
+
+  // return is serialized as JSON
+  return { foo: d, variable };
+}
+```
+
+## CLI
+
+We have a powerful CLI to interact with the windmill platform and sync your
+scripts from local files, github repos and to run scripts and flows on the
+instance from local commands. See
+[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
+
+![CLI Screencast](./cli/vhs/output/setup.gif)
+
+### Running scripts locally
+
+You can run your script locally easily, you simply need to pass the right
+environment variables for the `wmill` client library to fetch resources and
+variables from your instance if necessary. See more:
+<https://docs.windmill.dev/docs/advanced/local_development/>
 
 ## Stack
 
@@ -124,10 +144,11 @@ and much more!
   the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
+- golang runtime is 1.19.1
 
 ## Security
 
-### Sandboxing and workload isolation
+### Sandboxing
 
 Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
 sandboxing. It is production multi-tenant grade secure. Do not take our word for
@@ -143,43 +164,52 @@ That is what we do at <https://app.windmill.dev>.
 
 ## Performance
 
-The performances are great, as long as you do not exceed the parrallelism of the
-workers, we are
-[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
+Once a job started, there is no overhead compared to running the same script on
+the node with its corresponding runner (Deno/Go/Python/Bash). The added latency
+from a job being pulled from the queue, started, and then having its result sent
+back to the database is ~50ms. A typical lightweight deno job will take around
+100ms total.
 
 ## Architecture
 
 <p align="center">
 
-### Big-picture Architecture
-
 <img src="./imgs/diagram.svg">
 
-### Technical Architecture
-
-<img src="./imgs/architecture.svg">
-
 </p>
 
 ## How to self-host
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/how-tos/self_host)
+[documentation](https://docs.windmill.dev/docs/advanced/self_host)
 
 ### Docker compose
 
-`docker compose up` with the following docker-compose is sufficient:
-<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
+```
+curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
+curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
+curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
 
-For older kernels < 4.18, set `DISABLE_NUSER=true` as env variable, otherwise
-nsjail will not be able to launch the isolated scripts.
+docker compose up -d --pull always
+```
 
-To disable nsjail altogether, set `DISABLE_NSJAIL=true`.
+Go to http://localhost et voilà :)
 
 The default super-admin user is: admin@windmill.dev / changeme
 
-From there, you can create other users (do not forget to change the password!)
+From there, you can follow the setup app and creat other users.
+
+### Kubernetes (k8s) and Helm charts
+
+We publish helm charts at:
+<https://github.com/windmill-labs/windmill-helm-charts>
+
+### Postgres without superuser
+
+If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
+postgres superuser, you can run `./init-db-as-superuser.sql` to init the
+required users for windmill.
 
 ### Commercial license
 
@@ -191,14 +221,14 @@ comfortable with AGPLv3.
 To re-expose any Windmill parts to your users as a feature of your product, or
 to build a feature on top of Windmill, to comply with AGPLv3 your product must
 be AGPLv3 or you must get a commercial license. Contact us at
-<license@windmill.dev> if you have any doubts.
+<ruben@windmill.dev> if you have any doubts.
 
 In addition, a commercial license grants you a dedicated engineer to transition
 your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permissioning features
-such as groups and the ability to create more than one workspace.
+export features, SSO, unlimited users creation, advanced permission managing
+features such as groups and the ability to create more than one workspace.
 
-### OAuth for self-hosting (very optional)
+### OAuth for self-hosting
 
 To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
 the following format:
@@ -207,25 +237,104 @@ the following format:
 {
   "<client>": {
     "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>"
+    "secret": "<CLIENT_SECRET>",
+    "allowed_domains": ["windmill.dev"] //restrict a client OAuth login to some domains
   }
 }
 ```
 
 and mount it at `/usr/src/app/oauth.json`.
 
+The redirect url for the oauth clients is:
+`<instance_url>/user/login_callback/<client>`
+
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://docs.windmill.dev/docs/contributors_guide). We
+[Contributor's guide](https://docs.windmill.dev/docs/misc/contributing). We
 welcome contributions!
 
+You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
+
+```json
+{
+  "<client>": {
+    "id": "<CLIENT_ID>",
+    "secret": "<CLIENT_SECRET>",
+    // To add a new OAuth2 IdP
+    "login_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "userinfo_url": "<userinfo endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    },
+    // To add a new OAuth2 Resource
+    "connect_config": {
+      "auth_url": "<auth_endpoint>",
+      "token_url": "<token_endpoint>",
+      "scopes": ["scope1", "scope2"],
+      "extra_params": "<if_needed>"
+    }
+  }
+}
+```
+
 ### Resource types
 
 You will also want to import all the approved resource types from
-[WindmillHub](https://hub.windmill.dev). There is no automatic way to do this
-automatically currently, but it will be possible using a command with the
-upcoming CLI tool.
+[WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
+it being synced automatically everyday.
+
+## Environment Variables
+
+| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
+| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                   | 300                    | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| ZOMBIE_JOB_TIMEOUT        | 30                     | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
+| RESTART_ZOMBIE_JOBS       | true                   | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
+| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| GOPRIVATE                 |                        | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
+| NETRC                     |                        | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
+| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| NPM_CONFIG_REGISTRY       | None                   | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
+| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
+| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| INSTANCE_EVENTS_WEBHOOK   | None                   | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        | Server                |
 
 ## Run a local dev setup
 
@@ -246,9 +355,9 @@ See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
 running options.
 
 1. Create a Postgres Database for Windmill and create an admin role inside your
-   Postgres setup.
-   The easiest way to get a working postgres is running `cargo install sqlx-cli && sqlx migrate run`.
-   This will also avoid compile time issue with sqlx's `query!` macro
+   Postgres setup. The easiest way to get a working postgres is running
+   `cargo install sqlx-cli && sqlx migrate run`. This will also avoid compile
+   time issue with sqlx's `query!` macro
 2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
    your PATH
 3. Install deno and python3, have the bins at `/usr/bin/deno` and
@@ -257,7 +366,8 @@ running options.
 5. Install the [lld linker](https://lld.llvm.org/)
 6. Go to `frontend/`:
    1. `npm install`, `npm run generate-backend-client` then `npm run dev`
-   2. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will crash
+   2. In another shell `npm run build` otherwise the backend will not find the
+      `frontend/build` folder and will crash
    3. In another shell `sudo caddy run --config Caddyfile`
 7. Go to `backend/`:
    `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
@@ -271,4 +381,4 @@ running options.
 
 ## Copyright
 
-Windmill Labs, Inc 2022
+Windmill Labs, Inc 2023

backend/.cargo/config.toml

@@ -8,3 +8,14 @@ rustflags = [
 ]
 incremental = true
 
+[target.x86_64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup",
+]

backend/.gitignore

@@ -1,3 +1,4 @@
 target/
 .env
 oauth.json
+windmill-api/openapi-deref.yaml

backend/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.analysis.typeCheckingMode": "basic"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.43.0"
+version = "1.87.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.43.0"
+version = "1.87.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -27,6 +27,9 @@ edition = "2021"
 name = "windmill"
 path = "./src/main.rs"
 
+[features]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+
 [dependencies]
 anyhow.workspace = true
 tokio.workspace = true
@@ -41,12 +44,20 @@ sqlx.workspace = true
 tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
+git-version.workspace = true
+rsa.workspace = true
+base64.workspace = true
+sha2.workspace = true
+rsmq_async.workspace = true
+url.workspace = true
+
 
 [dev-dependencies]
 serde_json.workspace = true
 reqwest.workspace = true
 windmill-queue.workspace = true
 axum.workspace = true
+serde.workspace = true
 
 [workspace.dependencies]
 windmill-api = { path = "./windmill-api" }
@@ -59,12 +70,13 @@ windmill-parser = { path = "./parsers/windmill-parser" }
 windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
 windmill-parser-py = { path = "./parsers/windmill-parser-py" }
 windmill-parser-go = { path = "./parsers/windmill-parser-go" }
+windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
 axum = { version = "^0", features = ["headers"] }
 headers = "^0"
 hyper = { version = "^0", features = ["full"] }
 tokio = { version = "^1", features = ["full", "tracing"] }
 tower = "^0"
-tower-http = { version = "^0", features = ["trace"] }
+tower-http = { version = "^0", features = ["trace", "cors"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
@@ -72,10 +84,11 @@ uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"] }
+chrono-tz = "^0"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
-console-subscriber = "^0"
 prometheus = { version = "^0", default-features = false }
+cookie = { version = "0.17.0" }
 phf = { version = "0.11", features = ["macros"] }
 rust-embed = "^6"
 mime_guess = "^2"
@@ -110,10 +123,10 @@ itertools = "^0"
 regex = "^1"
 deno_core = "^0"
 async-recursion = "^1"
-swc_common = "^0"
-swc_ecma_parser = "^0"
-swc_ecma_ast = "^0"
-base64 = "^0"
+swc_common = "0.29.39"
+swc_ecma_parser = "0.128.2"
+swc_ecma_ast = "0.98.1"
+base64 = "0.21.0"
 unicode-general-category = "^0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
@@ -130,6 +143,20 @@ sqlx = { version = "^0", features = [
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
+futures-core = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
+const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
+dyn-iter = "0.2.0"
+rsa = "0.7.2"
+async-stripe = { version = "0.14", features = [
+    "runtime-tokio-hyper",
+    "checkout",
+] }
+async_zip = { version = "0.0.11", features = ["full"] }
+once_cell = "1.17.1"
+rsmq_async = { version = "5.1.5" }
+gosyn = "0.2.2"
+
+[patch.crates-io]

backend/build.rs

@@ -0,0 +1,5 @@
+// generated by `sqlx migrate build-script`
+fn main() {
+    // trigger recompilation when a new migration is added
+    println!("cargo:rerun-if-changed=migrations");
+}

backend/migrations/20221024225533_apps.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE app;
+DROP TABLE app_version;
+DROP TYPE EXECUTION_MODE;

backend/migrations/20221024225533_apps.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+CREATE TABLE app (
+    id BIGSERIAL PRIMARY KEY,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+	path varchar(255) NOT NULL,
+    summary VARCHAR(1000) NOT NULL DEFAULT '',
+    policy JSONB NOT NULL,
+    versions BIGINT[] NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE app_version(
+    id BIGSERIAL PRIMARY KEY,
+    flow_id BIGINT NOT NULL,
+    value JSONB NOT NULL,
+	created_by VARCHAR(50) NOT NULL,
+	created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    FOREIGN KEY (flow_id) REFERENCES app(id) ON DELETE CASCADE
+);
+
+CREATE POLICY see_own ON app FOR ALL
+USING (SPLIT_PART(app.path, '/', 1) = 'u' AND SPLIT_PART(app.path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON app FOR ALL
+USING (SPLIT_PART(app.path, '/', 1) = 'g' AND SPLIT_PART(app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20221104105107_queue-raw_lock.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE queue DROP raw_lock;

backend/migrations/20221104105107_queue-raw_lock.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN raw_lock TEXT DEFAULT NULL;

backend/migrations/20221104111317_completed_job-raw_lock.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE queue DROP raw_lock;

backend/migrations/20221104111317_completed_job-raw_lock.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE completed_job ADD COLUMN raw_lock TEXT DEFAULT NULL;

backend/migrations/20221104135246_flow-dependency_job.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE flow DROP dependency_job;

backend/migrations/20221104135246_flow-dependency_job.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE flow ADD COLUMN dependency_job UUID DEFAULT NULL;
+ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'flowdependencies';

backend/migrations/20221105003256_grant_all.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221105003256_grant_all.up.sql

@@ -0,0 +1,13 @@
+-- Add up migration script here
+
+
+GRANT ALL
+ON ALL TABLES IN SCHEMA public 
+TO windmill_user;
+
+
+GRANT ALL
+ON ALL TABLES IN SCHEMA public 
+TO windmill_admin;
+
+DELETE FROM workspace_invite WHERE workspace_id = 'demo' AND email = 'ruben@windmill.dev';

backend/migrations/20221106081530_bash.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221106081530_bash.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE 'bash';

backend/migrations/20221123151919_grant_sequence_privileges.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221123151919_grant_sequence_privileges.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_user;
+GRANT ALL
+ON ALL SEQUENCES IN SCHEMA public 
+TO windmill_admin;

backend/migrations/20221126181131_pg_rt_name.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221126181131_pg_rt_name.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+UPDATE resource_type rt SET name = 'postgresql' WHERE name = 'postgres' AND  NOT EXISTS (
+   SELECT 1 FROM resource_type WHERE name = 'postgresql' AND rt.workspace_id = workspace_id
+);
+
+UPDATE resource SET resource_type = 'postgresql' WHERE resource_type = 'postgres';

backend/migrations/20221127093847_add_account_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127093847_add_account_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE account ADD COLUMN refresh_error TEXT;

backend/migrations/20221127143814_remove_oauth_from_res.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221127143814_remove_oauth_from_res.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE resource DROP COLUMN is_oauth;

backend/migrations/20221128210151_favorites.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221128210151_favorites.up.sql

@@ -0,0 +1,11 @@
+-- Add up migration script here
+
+CREATE TYPE FAVORITE_KIND AS ENUM ('app', 'script', 'flow');
+
+CREATE TABLE favorite (
+    usr VARCHAR(50) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    path VARCHAR(255) NOT NULL,
+    favorite_kind FAVORITE_KIND NOT NULL,
+    PRIMARY KEY (usr, workspace_id, favorite_kind, path)
+);

backend/migrations/20221203080006_apps_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221203080006_apps_2.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE app ADD CONSTRAINT unique_path_workspace_id UNIQUE (workspace_id, path);
+ALTER TABLE app ENABLE ROW LEVEL SECURITY;

backend/migrations/20221207103910_admins_workspace.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207103910_admins_workspace.up.sql

@@ -0,0 +1,9 @@
+INSERT INTO workspace(id, name, owner) VALUES
+	('admins', 'Admins', 'admin@windmill.dev');
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('admins');
+
+INSERT INTO workspace_key
+	(workspace_id, kind, key)
+	VALUES ('admins', 'cloud', md5(random()::text) || md5(random()::text))

backend/migrations/20221207125219_admins_hub_sync.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221207125219_admins_hub_sync.up.sql

@@ -0,0 +1,20 @@
+INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'admins',
+'system',
+'import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
+
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'Syncronize Hub Resource types with starter workspace',
+'Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.',
+'u/admin/hub_sync', -28028598712388162, 'deno', '');

backend/migrations/20221208135921_job_pre_run_error.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221208135921_job_pre_run_error.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN pre_run_error text;

backend/migrations/20221210125521_increase_variable_length.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210125521_increase_variable_length.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000);

backend/migrations/20221210194626_usage_tracker.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221210194626_usage_tracker.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE usage (
+    id VARCHAR(50) NOT NULL,
+    is_workspace BOOLEAN NOT NULL,
+    month_ INTEGER NOT NULL,
+    usage INTEGER NOT NULL,
+    PRIMARY KEY (id, is_workspace, month_)
+);

backend/migrations/20221211055346_emails_everywhere.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN email; 
+ALTER TABLE workspace_settings DROP COLUMN slack_email; 
+ALTER TABLE schedule DROP COLUMN email; 
+ALTER TABLE schedule DROP COLUMN error; 

backend/migrations/20221211055346_emails_everywhere.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 
+ALTER TABLE workspace_settings ADD COLUMN slack_email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';;
+ALTER TABLE schedule ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz';  
+ALTER TABLE schedule ADD COLUMN error TEXT;  
+

backend/migrations/20221211192539_grant_usage.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_domain;

backend/migrations/20221211192539_grant_usage.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_admin;
+GRANT ALL PRIVILEGES ON TABLE usage TO windmill_user;

backend/migrations/20221211200923_remove_admin_from_starters.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221211200923_remove_admin_from_starters.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+DELETE FROM usr WHERE workspace_id = 'starter' AND email = 'admin@windmill.dev';

backend/migrations/20221212114409_add_all_group_to_admins.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212114409_add_all_group_to_admins.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO group_ SELECT id, 'all', 'The group that always contains all users of this workspace' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20221212130731_add_autoinvite.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221212130731_add_autoinvite.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_domain VARCHAR(50);
+ALTER TABLE workspace DROP COLUMN domain;

backend/migrations/20221213112420_add_operator.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE workspace_invite DROP COLUMN operator;
+ALTER TABLE workspace_settings DROP COLUMN auto_invite_operator;

backend/migrations/20221213112420_add_operator.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE workspace_invite ADD COLUMN operator BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE workspace_settings ADD COLUMN auto_invite_operator BOOLEAN DEFAULT false;
+ALTER TABLE completed_job ADD COLUMN email VARCHAR(50) NOT NULL DEFAULT 'missing@email.xyz'; 

backend/migrations/20221214105402_first_time_users.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221214105402_first_time_users.up.sql

@@ -0,0 +1,12 @@
+-- Add up migration script here
+ALTER TABLE
+    password
+ADD
+    first_time_user boolean NOT NULL DEFAULT (false);
+
+UPDATE
+    password
+SET
+    first_time_user = true
+WHERE
+    email = 'admin@windmill.dev';

backend/migrations/20221215101605_share_job_to_owner.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE queue DROP COLUMN visible_to_owner;
+ALTER TABLE completed_job DROP COLUMN visible_to_owner;
+
+DROP POLICY see_own_path ON queue;

backend/migrations/20221215101605_share_job_to_owner.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+ALTER TABLE queue ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+ALTER TABLE completed_job ADD COLUMN visible_to_owner BOOLEAN DEFAULT true;
+
+CREATE POLICY see_own_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'u' AND SPLIT_PART(queue.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON queue FOR ALL
+USING (queue.visible_to_owner IS true AND SPLIT_PART(queue.script_path, '/', 1) = 'g' AND SPLIT_PART(queue.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_own_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'u' AND SPLIT_PART(completed_job.script_path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member_path ON completed_job FOR ALL
+USING (completed_job.visible_to_owner IS true AND SPLIT_PART(completed_job.script_path, '/', 1) = 'g' AND SPLIT_PART(completed_job.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));

backend/migrations/20221219032311_folders.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20221219032311_folders.up.sql

@@ -0,0 +1,19 @@
+-- Add up migration script here
+ALTER TABLE app_version
+RENAME COLUMN flow_id TO app_id;
+
+CREATE TABLE folder (
+    name VARCHAR(255),
+    workspace_id VARCHAR(50) NOT NULL,
+    display_name VARCHAR(100) NOT NULL,
+    owners VARCHAR(255)[] NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}',
+    FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE,
+    PRIMARY KEY (workspace_id, name)
+);
+
+CREATE INDEX folder_extra_perms ON folder USING GIN (extra_perms);
+CREATE INDEX folder_owners ON folder USING GIN (owners);
+
+GRANT ALL ON folder TO windmill_user;
+GRANT ALL ON folder TO windmill_admin;

backend/migrations/20221219051235_folders_2.down.sql

@@ -0,0 +1,21 @@
+-- Add down migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        DROP POLICY see_folder_extra_perms_user ON %1$I;
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+
+DROP POLICY see_extra_perms_user ON folder;
+DROP POLICY see_extra_perms_groups ON folder;

backend/migrations/20221219051235_folders_2.up.sql

@@ -0,0 +1,52 @@
+-- Add up migration script here
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'app'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL
+        USING (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+        WITH CHECK (SPLIT_PART(%1$I.path, '/', 1) = 'f' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;
+ALTER TABLE folder ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY see_extra_perms_user ON folder FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')) or (CONCAT('u/', current_setting('session.user')) = ANY(owners)))
+WITH CHECK ((CONCAT('u/', current_setting('session.user')) = ANY(owners)));
+
+CREATE POLICY see_extra_perms_groups ON folder FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[] or (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]))))
+WITH CHECK (exists(
+    SELECT o FROM unnest(owners) as o
+    WHERE o = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])));
+
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['script', 'flow', 'variable', 'resource', 'schedule'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        ALTER TABLE %1$I DROP CONSTRAINT proper_id;
+        ALTER TABLE %1$I ADD CONSTRAINT proper_id CHECK (path ~ '^[ufg](\/[\w-]+){2,}$');
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;

backend/migrations/20230101074836_sync_from_admin.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230101074836_sync_from_admin.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.57.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230107075643_use_internal_url_sync.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230107075643_use_internal_url_sync.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.57.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with admins workspace',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230119194229_customer_id.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230119194229_customer_id.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN customer_id VARCHAR(100);
+ALTER TABLE workspace_settings ADD COLUMN plan VARCHAR(40);

backend/migrations/20230126023323_webhook.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230126023323_webhook.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE
+    workspace_settings
+ADD
+    COLUMN webhook text;

backend/migrations/20230127175428_increase_token_size.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230127175428_increase_token_size.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE account ALTER COLUMN refresh_token TYPE VARCHAR(1500);

backend/migrations/20230127210757_favorite_privilege.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230127210757_favorite_privilege.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+GRANT ALL PRIVILEGES ON TABLE favorite TO windmill_admin;
+GRANT ALL PRIVILEGES ON TABLE favorite TO windmill_user;

backend/migrations/20230131143413_more_folders_privilege.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230131143413_more_folders_privilege.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+
+
+CREATE POLICY see_folder_extra_perms_user ON capture FOR ALL
+USING (SPLIT_PART(capture.path, '/', 1) = 'f' AND SPLIT_PART(capture.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(capture.path, '/', 1) = 'f' AND SPLIT_PART(capture.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+DO
+$do$
+  DECLARE
+    i text;
+    arr text[] := array['queue', 'completed_job'];
+  BEGIN
+  FOREACH i IN ARRAY arr
+  LOOP
+    EXECUTE FORMAT(
+      $$
+        CREATE POLICY see_folder_extra_perms_user ON %1$I FOR ALL
+        USING (%1$I.visible_to_owner IS true AND SPLIT_PART(%1$I.script_path, '/', 1) = 'f' AND SPLIT_PART(%1$I.script_path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
+      $$,
+      i
+    );
+  END LOOP;
+  END
+$do$;