fix: benchmark action single push (#3480)

* fix: benchmarks action only push once

* fix: benchmark artifacts

* fix: include past data

* fix: don't save dedicated warmup

* fix: fetch existing data directly from gh

.github/workflows/benchmark.yml

@@ -34,24 +34,18 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update single worker benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_single
+          path: |
+            *.json
 
   benchmark_dedicated:
     runs-on: ubicloud-standard-8
@@ -81,24 +75,18 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
           --no-warm-up -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update dedicated worker benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_dedicated
+          path: |
+            *.json
 
   benchmark_4workers:
     runs-on: ubicloud-standard-8
@@ -161,9 +149,6 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
@@ -171,15 +156,12 @@ jobs:
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
           --workers 4
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update 4 workers benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_4workers
+          path: |
+            *.json
 
   benchmark_8workers:
     runs-on: ubicloud-standard-8
@@ -285,9 +267,6 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
@@ -295,15 +274,12 @@ jobs:
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
           --workers 8
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update 8 workers benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_8workers
+          path: |
+            *.json
 
   benchmark_graphs:
     runs-on: ubicloud
@@ -319,6 +295,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: benchmarks
+      - name: Download benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
       - name: graphs
         run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts
@@ -326,10 +306,10 @@ jobs:
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json
       - name: Push changes
         run: |
+          ls -la
           pwd
-          git pull origin benchmarks
           git add .
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
-          git commit -m "Update graphs"
+          git commit -m "Update benchmarks"
           git push

benchmarks/benchmark_suite.ts

@@ -9,6 +9,7 @@ import { VERSION } from "./lib.ts";
 type Config = {
   kind: string;
   jobs: number;
+  noSave?: boolean;
 }[];
 
 async function warmUp(
@@ -79,6 +80,10 @@ async function main({
           jobs: benchmark.jobs,
         });
 
+        if (benchmark.noSave) {
+          continue;
+        }
+
         if (!result) {
           throw new Error("No result returned");
         }
@@ -91,8 +96,10 @@ async function main({
           benchmark.kind + (workers > 1 ? `_${workers}workers` : "");
         const jsonFilePath = `${benchmarkName}_benchmark.json`;
         try {
-          const existing = await Deno.readTextFile(jsonFilePath);
-          data = JSON.parse(existing);
+          const remotePath =
+            "https://raw.githubusercontent.com/windmill-labs/windmill/benchmarks/" +
+            jsonFilePath;
+          data = await fetch(remotePath).then((r) => r.json());
         } catch (_) {
           console.log("No existing data file found, creating new one.");
         }

benchmarks/suite_config.json

@@ -1,41 +1,33 @@
 [
   {
-    "graph_title": "noop throughput benchmark",
     "kind": "noop",
     "jobs": 90000
   },
   {
-    "graph_title": "flow throughput benchmark",
     "kind": "2steps",
     "jobs": 250
   },
   {
-    "graph_title": "deno throughput benchmark",
     "kind": "deno",
     "jobs": 500
   },
   {
-    "graph_title": "bun throughput benchmark",
     "kind": "bun",
     "jobs": 500
   },
   {
-    "graph_title": "python throughput benchmark",
     "kind": "python",
     "jobs": 500
   },
   {
-    "graph_title": "go throughput benchmark",
     "kind": "go",
     "jobs": 2000
   },
   {
-    "graph_title": "bash throughput benchmark",
     "kind": "bash",
     "jobs": 2000
   },
   {
-    "graph_title": "nativets throughput benchmark",
     "kind": "nativets",
     "jobs": 2000
   }

benchmarks/suite_dedicated.json

@@ -1,11 +1,10 @@
 [
   {
-    "graph_title": "warmup",
     "kind": "dedicated",
-    "jobs": 50000
+    "jobs": 50000,
+    "noSave": true
   },
   {
-    "graph_title": "dedicated throughput benchmark",
     "kind": "dedicated",
     "jobs": 90000
   }