windmill/.github/workflows/benchmark.yml

name: Run benchmarks

on:
  schedule:
    - cron: "0 0 */1 * *"
  workflow_dispatch:

jobs:
  benchmark_single:
    runs-on: ubicloud-standard-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
          --shm-size=2g


      windmill:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always --health-interval 10s --health-timeout 5s
          --health-retries 5 --health-cmd "curl
          http://localhost:8000/api/version"
        ports:
          - 8000:8000
    steps:
      - uses: denoland/setup-deno@v2
        with:
          deno-version: v2.x
      - name: benchmark
        timeout-minutes: 30
        run: deno run  -A -r
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
          -c
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
      - name: Save benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark_single
          path: |
            *.json

  benchmark_dedicated:
    runs-on: ubicloud-standard-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
      windmill:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          WORKER_GROUP: dedicated
          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
        options: >-
          --pull always --restart unless-stopped --health-interval 10s --health-timeout 5s
          --health-retries 5 --health-cmd "curl
          http://localhost:8000/api/version"
        ports:
          - 8000:8000
    steps:
      - uses: denoland/setup-deno@v2
        with:
          deno-version: v2.x
      - name: benchmark
        timeout-minutes: 20
        run: deno run  -A -r
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
          --no-warm-up -c
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
      - name: Save benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark_dedicated
          path: |
            *.json

  benchmark_4workers:
    runs-on: ubicloud-standard-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
      windmill:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always --health-interval 10s --health-timeout 5s
          --health-retries 5 --health-cmd "curl
          http://localhost:8000/api/version"
        ports:
          - 8000:8000
      windmill_1:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_2:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_3:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

    steps:
      - uses: denoland/setup-deno@v2
        with:
          deno-version: v2.x
      - name: benchmark
        timeout-minutes: 20
        run: deno run  -A -r
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
          -c
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
          --workers 4
          --factor 3
      - name: Save benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark_4workers
          path: |
            *.json

  benchmark_8workers:
    runs-on: ubicloud-standard-8
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
      windmill:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always --health-interval 10s --health-timeout 5s
          --health-retries 5 --health-cmd "curl
          http://localhost:8000/api/version"
        ports:
          - 8000:8000
      windmill_1:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_2:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_3:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_4:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_5:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_6:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always

      windmill_7:
        image: ghcr.io/windmill-labs/windmill-ee:main
        env:
          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
          MODE: worker
          WORKER_GROUP: main
          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
        options: >-
          --pull always
    steps:
      - uses: denoland/setup-deno@v2
        with:
          deno-version: v2.x
      - name: benchmark
        timeout-minutes: 20
        run: deno run  -A -r
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
          -c
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
          --workers 8
          --factor 3
      - name: Save benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark_8workers
          path: |
            *.json

  benchmark_graphs:
    runs-on: ubicloud
    needs:
      - benchmark_single
      - benchmark_dedicated
      - benchmark_4workers
      - benchmark_8workers
    steps:
      - uses: denoland/setup-deno@v2
        with:
          deno-version: v2.x
      - uses: actions/checkout@v4
        with:
          ref: benchmarks
      - name: Download benchmark results
        uses: actions/download-artifact@v4
        with:
          merge-multiple: true
      - name: graphs
        run: deno run -A -r
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts
          -c
          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json
      - name: Push changes
        run: |
          ls -la
          pwd
          git add .
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git commit -m "Update benchmarks"
          git push