diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index d7930e26b5..e774f7d454 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -89,7 +89,7 @@ jobs:
         run:
           deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          -c
+          --no-warm-up -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
 
   benchmark_4workers:
@@ -125,9 +125,7 @@ jobs:
           WORKER_GROUP: main
           WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
         options: >-
-          --pull always --health-interval 10s --health-timeout 5s
-          --health-retries 5 --health-cmd "curl
-          http://localhost:8000/api/version"
+          --pull always
 
       windmill_2:
         image: ghcr.io/windmill-labs/windmill-ee:main
@@ -138,9 +136,7 @@ jobs:
           WORKER_GROUP: main
           WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
         options: >-
-          --pull always --health-interval 10s --health-timeout 5s
-          --health-retries 5 --health-cmd "curl
-          http://localhost:8000/api/version"
+          --pull always
 
       windmill_3:
         image: ghcr.io/windmill-labs/windmill-ee:main
@@ -151,9 +147,7 @@ jobs:
           WORKER_GROUP: main
           WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
         options: >-
-          --pull always --health-interval 10s --health-timeout 5s
-          --health-retries 5 --health-cmd "curl
-          http://localhost:8000/api/version"
+          --pull always
 
     steps:
       - uses: denoland/setup-deno@v1
diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts
index d81f7ec794..a372b3b5ea 100644
--- a/benchmarks/benchmark_suite.ts
+++ b/benchmarks/benchmark_suite.ts
@@ -66,7 +66,9 @@ async function main({
     }
   }
 
-  await warmUp(host, email, password, token, workspace);
+  if (!Deno.args.includes("--no-warm-up")) {
+    await warmUp(host, email, password, token, workspace);
+  }
 
   try {
     const config = await getConfig(configPath);
@@ -180,6 +182,7 @@ await new Command()
   .option("-c --config-path <config:string>", "The path of the config file", {
     required: true,
   })
+  .option("--no-warm-up", "Skip the warm up phase.")
   .action(main)
   .command(
     "upgrade",