diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 195821b2dd..d420ff1f00 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -290,6 +290,49 @@ jobs: path: | *.json + benchmark_wac: + runs-on: ubicloud-standard-8 + services: + postgres: + image: postgres + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + --shm-size=2g + windmill: + image: ghcr.io/windmill-labs/windmill-ee:main + env: + DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill + LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + WORKER_GROUP: main + WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + options: >- + --pull always --health-interval 10s --health-timeout 5s + --health-retries 5 --health-cmd "curl + http://localhost:8000/api/version" + ports: + - 8000:8000 + steps: + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: benchmark + timeout-minutes: 30 + run: deno run -A -r + https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts + -c + https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_wac.json + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_wac + path: | + *.json + benchmark_graphs: runs-on: ubicloud needs: @@ -297,6 +340,7 @@ jobs: - benchmark_dedicated - benchmark_4workers - benchmark_8workers + - benchmark_wac steps: - uses: denoland/setup-deno@v2 with: diff --git a/benchmarks/Dockerfile b/benchmarks/Dockerfile index c8f3fe83d5..7655e5f2d2 100644 --- a/benchmarks/Dockerfile +++ b/benchmarks/Dockerfile @@ -1,14 +1,20 @@ -FROM denoland/deno:alpine-1.26.2 +FROM denoland/deno:alpine-2.1.4 WORKDIR /app USER deno +ADD ./lib.ts . +ADD ./action.ts . ADD ./main.ts . -RUN deno cache --unstable main.ts +RUN deno cache main.ts ADD ./worker.ts . -RUN deno cache --unstable worker.ts +RUN deno cache worker.ts ADD ./scraper.ts . -RUN deno cache --unstable scraper.ts +RUN deno cache scraper.ts +ADD ./benchmark_oneoff.ts . +RUN deno cache benchmark_oneoff.ts +ADD ./benchmark_suite.ts . +RUN deno cache benchmark_suite.ts -ENTRYPOINT [ "/tini", "--", "docker-entrypoint.sh", "run", "--unstable", "-A", "main.ts" ] \ No newline at end of file +ENTRYPOINT [ "/tini", "--", "docker-entrypoint.sh", "run", "-A", "main.ts" ] diff --git a/benchmarks/README.md b/benchmarks/README.md index c358471626..fc758c006f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,85 +1,71 @@ # Benchmarks -This folder includes a small deno/ts utility to benchmark execution of jobs & -flows. +Deno/TS benchmark suite for measuring Windmill job and flow execution throughput. -## Installation +## Quick Start -Install the `wmill` CLI tool using -`deno install --unstable -A https://deno.land/x/wmillbench/main.ts`. +```bash +# Install Deno +curl -fsSL https://deno.land/install.sh | sh -Update to the latest version using `wmillbench upgrade`. +# Run a single benchmark +deno run -A benchmark_oneoff.ts --kind noop --jobs 10000 -To build a local version, you can just run: -``` -deno install -A main.ts +# Run the full suite +deno run -A benchmark_suite.ts -c suite_config.json + +# Run WAC v2 benchmarks (workflow-as-code vs flow comparison) +deno run -A benchmark_suite.ts -c suite_wac.json ``` -## Quickstart +## Benchmark Kinds -Have your instance expose prometheus metrics (METRICS_ADDR=true). +### Script benchmarks +- `noop` — Empty jobs (measures pure scheduling overhead) +- `deno`, `bun`, `python`, `go`, `bash` — Language runtimes +- `nativets` — BunNative (no isolation) +- `dedicated`, `dedicated_nativets` — Dedicated worker mode -Then +### Flow benchmarks +- `2steps` — 2-step flow (deno + identity) +- `bigscriptinflow` — Flow with large raw bash script +- `flow_seq_2_bun` — 2 sequential bun steps +- `flow_par_2_bun` — 2 parallel bun steps (branchall) +- `flow_seq_3_bun` — 3 sequential bun steps +- `flow:` — Custom flow by path +- `script:` — Custom script by path -``` -wmillbench -e admin@windmill.dev -p changeme --host YOUR_HOST +### WAC v2 benchmarks (workflow-as-code) +- `wac_seq_2` — 2 sequential tasks +- `wac_par_2` — 2 parallel tasks (Promise.all) +- `wac_seq_3` — 3 sequential tasks +- `wac_inline_2` — 2 inline steps (no child jobs) + +## Suite Configs + +| File | Description | +|------|-------------| +| `suite_config.json` | Main benchmark suite (noop, languages, flows) | +| `suite_dedicated.json` | Dedicated worker benchmarks | +| `suite_dedicated_nativets.json` | Dedicated NativeTS benchmarks | +| `suite_wac.json` | WAC v2 vs flow comparison benchmarks | + +## Interactive Benchmark Tool + +```bash +deno run -A main.ts -e admin@windmill.dev -p changeme --host http://localhost:8000 ``` -## Usage +Options: `--workers`, `--seconds`, `--maximum-throughput`, `--use-flows`, `--script-pattern`, `--export-json`, `--export-csv` -Usage: wmillbench +## Graph Generation -Description: - -Run Benchmark to measure throughput of windmill. - -Options: - --h, --help - Show this help. --V, --version - Show the version number for this program. ---host - The windmill host to benchmark. (Default: "http://127.0.0.1:8000/") ---workers - The number of workers to run at once. (Default: 1) --s, --seconds - How long to run the benchmark for (in seconds). (Default: 30) --e, --email - The email to use to login. --p, --password - The password to use to login. --t, --token - The token to use when talking to the API server. Preferred over manual login. --w, --workspace - The workspace to spawn scripts from. (Default: "starter") --m, --metrics - The url to scrape metrics from. (Default: "http://localhost:8001/metrics") ---export-json - If set, exports will be into a JSON file. ---export-csv - If set, exports will be into a csv file. ---export-histograms [histograms...] - Mark metrics (without label) that are reported as histograms to export. ---export-simple [simple...] - Mark metrics (without label) that are reported as simple values. ---maximum-throughput - Maximum number of jobs/flows to start in one second. (Default: Infinity) ---use-flows - Run flows instead of jobs. ---histogram-buckets [buckets...] - Define what buckets to collect from histograms. (Default: [ "+Inf", "10", "5", "2.5", "2.5", "1", "0.5", "0.25", "0.1", "0.05", "0.025", "0.01", "0.005" ]) - -Environment variables: - -WM_TOKEN - The token to use when talking to the API server. Preferred -over manual login. WM_WORKSPACE - The workspace to spawn scripts -from. - - - -This will run a simple benchmark against localhost (the default admin email + -password are set above), all execution is done in the "bench" workspace (as set -via `--workspace`). - -Metrics are exported to JSON will only include mean & stdev, histograms get one -entry for each bucket. CSV will include a full list of all values scraped. - -## NOOP jobs benchmark - -A specific benchmark creating a set of NOOP jobs all at once in windmill is also available. -in `benchmarks_noop.ts` - -You can build it locally with: -``` -deno install -A benchmarks_noop.ts -``` -and then -``` -benchmarks_noop -e admin@windmill.dev -p changeme --host YOUR_HOST +```bash +deno run -A benchmark_graphs.ts -c graphs_config.json ``` -By default it creates 10000 jobs in Windmill in a single batch, but this is parametrizable. \ No newline at end of file +Generates SVG graphs from `*_benchmark.json` data files. + +## CI + +The GitHub Actions workflow (`.github/workflows/benchmark.yml`) runs hourly with 1/4/8 worker configurations plus WAC benchmarks. Results are committed to the `benchmarks` branch. diff --git a/benchmarks/benchmark_graphs.ts b/benchmarks/benchmark_graphs.ts index 23b3b763f7..bfaad7dbb6 100644 --- a/benchmarks/benchmark_graphs.ts +++ b/benchmarks/benchmark_graphs.ts @@ -3,32 +3,20 @@ import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgra import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; import { drawGraph, drawGraphMulti } from "./graph.ts"; -import { VERSION } from "./lib.ts"; +import { VERSION, loadJsonConfig } from "./lib.ts"; -type GraphsConfig = [ - { - graph_title: string; - benchmarks: { - kind: string; - workers: number; - label: string; - }[]; - jobs: number; - } -]; +type GraphsConfig = { + graph_title: string; + benchmarks: { + kind: string; + workers: number; + label: string; + }[]; +}[]; async function main({ configPath }: { configPath: string }) { - async function getConfig(configPath: string): Promise { - if (configPath.startsWith("http")) { - const response = await fetch(configPath); - return await response.json(); - } else { - return JSON.parse(await Deno.readTextFile(configPath)); - } - } - try { - const config = await getConfig(configPath); + const config = await loadJsonConfig(configPath); for (const graphConfig of config || []) { const data: { @@ -81,7 +69,7 @@ async function main({ configPath }: { configPath: string }) { } await new Command() - .name("wmillbenchsuite") + .name("wmillbenchgraphs") .description("Create and save graphs from benchmark data.") .version(VERSION) .option("-c --config-path ", "The path of the config file", { diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 0cd4c3483f..5f075d2034 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -10,7 +10,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -import { VERSION, createBenchScript, getFlowPayload, login } from "./lib.ts"; +import { VERSION, createBenchScript, createWacBenchScript, getFlowPayload, login, WAC_KINDS, STEPS_PER_WORKFLOW } from "./lib.ts"; async function verifyOutputs(uuids: string[], workspace: string) { console.log("Verifying outputs"); @@ -38,6 +38,8 @@ async function verifyOutputs(uuids: string[], workspace: string) { } export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] + +const FLOW_COMPARISON_KINDS = ["flow_seq_2_bun", "flow_par_2_bun", "flow_seq_3_bun"]; export async function main({ host, email, @@ -151,6 +153,8 @@ export async function main({ ) ) { await createBenchScript(kind, workspace); + } else if (WAC_KINDS.includes(kind)) { + await createWacBenchScript(kind, workspace); } @@ -173,6 +177,20 @@ export async function main({ kind: "script", path: "f/benchmarks/" + kind, }); + } else if (WAC_KINDS.includes(kind)) { + // WAC v2 scripts are deployed as bun scripts, run via script path + nStepsFlow = STEPS_PER_WORKFLOW[kind] ?? 0; + body = JSON.stringify({ + kind: "script", + path: "f/benchmarks/" + kind, + }); + } else if (FLOW_COMPARISON_KINDS.includes(kind)) { + nStepsFlow = STEPS_PER_WORKFLOW[kind] ?? 0; + const payload = getFlowPayload(kind); + body = JSON.stringify({ + kind: "flow", + flow_value: payload.value, + }); } else if (["2steps", "bigscriptinflow"].includes(kind)) { nStepsFlow = kind == "2steps" ? 2 : 1; const payload = getFlowPayload(kind); @@ -182,7 +200,7 @@ export async function main({ }); } else if (kind.startsWith("flow:")) { console.log("Detected custom flow "); - let flow_path = kind.substr(5); + let flow_path = kind.substring(5); nStepsFlow = await getFlowStepCount(config.workspace_id, flow_path); console.log(`Total steps of flow including sub-flows: ${nStepsFlow}`); body = JSON.stringify({ @@ -193,7 +211,7 @@ export async function main({ console.log("Detected custom script"); body = JSON.stringify({ kind: "script", - path: kind.substr(7), + path: kind.substring(7), }); } else if (kind == "bigrawscript") { noVerify = true; @@ -281,6 +299,9 @@ export async function main({ let lastElapsed = 0; let lastCompletedJobs = 0; + // Timeout: 10 minutes for the polling loop to prevent hanging forever + // (e.g. if WAC suspend/resume fails or jobs get stuck) + const POLL_TIMEOUT_MS = 10 * 60 * 1000; let didStart = false; while (completedJobs < jobsSent) { const loopStart = Date.now(); @@ -292,6 +313,10 @@ export async function main({ } } else { const elapsed = start ? Date.now() - start : 0; + if (elapsed > POLL_TIMEOUT_MS) { + console.error(`\nTimeout: benchmark did not complete within ${POLL_TIMEOUT_MS / 1000}s (${completedJobs}/${jobsSent} completed)`); + break; + } completedJobs = await getCompletedJobsCount(NON_TEST_TAGS); if (nStepsFlow > 0) { completedJobs = Math.floor(completedJobs / (nStepsFlow + 1)); @@ -338,7 +363,9 @@ export async function main({ kind !== "nativets" && kind !== "dedicated_nativets" && !kind.startsWith("flow:") && - !kind.startsWith("script:") + !kind.startsWith("script:") && + !WAC_KINDS.includes(kind) && + !FLOW_COMPARISON_KINDS.includes(kind) ) { await verifyOutputs(uuids, config.workspace_id); } @@ -387,7 +414,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets", + "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", { required: true, } diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index f4840dda05..caba48aa72 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -4,7 +4,7 @@ import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upg import { main as runBenchmark } from "./benchmark_oneoff.ts"; -import { VERSION } from "./lib.ts"; +import { VERSION, loadJsonConfig } from "./lib.ts"; type Config = { kind: string; @@ -50,21 +50,12 @@ async function main({ workers: number; factor?: number; }) { - async function getConfig(configPath: string): Promise { - if (configPath.startsWith("http")) { - const response = await fetch(configPath); - return await response.json(); - } else { - return JSON.parse(await Deno.readTextFile(configPath)); - } - } - if (!Deno.args.includes("--no-warm-up")) { await warmUp(host, email, password, token, workspace); } try { - const config = await getConfig(configPath); + const config = await loadJsonConfig(configPath); for (const benchmark of config) { try { console.log( diff --git a/benchmarks/graphs_config.json b/benchmarks/graphs_config.json index 174990ecb2..8e37695d66 100644 --- a/benchmarks/graphs_config.json +++ b/benchmarks/graphs_config.json @@ -223,5 +223,75 @@ "label": "noop" } ] + }, + { + "graph_title": "WAC v2 sequential vs flow sequential (2 steps, bun)", + "benchmarks": [ + { + "kind": "wac_seq_2", + "workers": 1, + "label": "WAC v2 sequential" + }, + { + "kind": "flow_seq_2_bun", + "workers": 1, + "label": "Flow sequential" + } + ] + }, + { + "graph_title": "WAC v2 parallel vs flow parallel (2 steps, bun)", + "benchmarks": [ + { + "kind": "wac_par_2", + "workers": 1, + "label": "WAC v2 parallel" + }, + { + "kind": "flow_par_2_bun", + "workers": 1, + "label": "Flow parallel" + } + ] + }, + { + "graph_title": "WAC v2 sequential vs flow sequential (3 steps, bun)", + "benchmarks": [ + { + "kind": "wac_seq_3", + "workers": 1, + "label": "WAC v2 sequential" + }, + { + "kind": "flow_seq_3_bun", + "workers": 1, + "label": "Flow sequential" + } + ] + }, + { + "graph_title": "WAC v2 patterns comparison", + "benchmarks": [ + { + "kind": "wac_seq_2", + "workers": 1, + "label": "sequential 2-task" + }, + { + "kind": "wac_par_2", + "workers": 1, + "label": "parallel 2-task" + }, + { + "kind": "wac_seq_3", + "workers": 1, + "label": "sequential 3-task" + }, + { + "kind": "wac_inline_2", + "workers": 1, + "label": "inline 2-step" + } + ] } ] \ No newline at end of file diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 829bd04e08..ac0e20f064 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -132,6 +132,119 @@ export async function createBenchScript( } } +// WAC v2 benchmark script content patterns +const WAC_SCRIPTS: Record = { + wac_seq_2: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "export const main = workflow(async () => {", + " const a = await step_a();", + " const b = await step_b();", + " return { a, b };", + "});", + ].join("\n"), + + wac_par_2: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "export const main = workflow(async () => {", + " const [a, b] = await Promise.all([step_a(), step_b()]);", + " return { a, b };", + "});", + ].join("\n"), + + wac_seq_3: [ + 'import { task, workflow } from "windmill-client";', + "const step_a = task(async () => { return 1; });", + "const step_b = task(async () => { return 2; });", + "const step_c = task(async () => { return 3; });", + "export const main = workflow(async () => {", + " const a = await step_a();", + " const b = await step_b();", + " const c = await step_c();", + " return { a, b, c };", + "});", + ].join("\n"), + + wac_inline_2: [ + 'import { step, workflow } from "windmill-client";', + "export const main = workflow(async () => {", + ' const a = await step("a", () => 1);', + ' const b = await step("b", () => 2);', + " return { a, b };", + "});", + ].join("\n"), +}; + +export const WAC_KINDS = Object.keys(WAC_SCRIPTS); + +// Number of child jobs created per workflow instance (used to compute throughput) +// For task(): each task creates a child job. For step(): no child job. +// Total completed jobs per workflow = nSteps + 1 (children + parent) +export const STEPS_PER_WORKFLOW: Record = { + wac_seq_2: 2, + wac_par_2: 2, + wac_seq_3: 3, + wac_inline_2: 0, // inline steps don't create child jobs + flow_seq_2_bun: 2, + flow_par_2_bun: 2, + flow_seq_3_bun: 3, +}; + +export async function createWacBenchScript( + wacPattern: string, + workspace: string, +) { + const scriptContent = WAC_SCRIPTS[wacPattern]; + if (!scriptContent) { + throw new Error("Unknown WAC pattern: " + wacPattern); + } + + const path = `f/benchmarks/${wacPattern}`; + const exists = await windmill.ScriptService.existsScriptByPath({ + workspace, + path, + }); + + if (exists) { + await windmill.ScriptService.deleteScriptByPath({ + workspace, + path, + }); + } + + const hash = await windmill.ScriptService.createScript({ + workspace, + requestBody: { + path, + content: scriptContent, + summary: wacPattern + " WAC v2 benchmark", + description: "", + language: "bun" as api.NewScript.language, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: {}, + required: [], + type: "object", + }, + }, + }); + + await waitForDeployment(workspace, hash); + console.log("Created WAC v2 benchmark script at path", path); +} + +export async function loadJsonConfig(configPath: string): Promise { + if (configPath.startsWith("http")) { + const response = await fetch(configPath); + return await response.json(); + } else { + return JSON.parse(await Deno.readTextFile(configPath)); + } +} + export const getFlowPayload = (flowPattern: string): api.FlowPreview => { if (flowPattern == "branchone") { return { @@ -260,6 +373,113 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { ], }, }; + } else if (flowPattern == "flow_seq_2_bun") { + return { + path: "flow_seq_2_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + ], + }, + }; + } else if (flowPattern == "flow_par_2_bun") { + return { + path: "flow_par_2_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + type: "branchall", + parallel: true, + branches: [ + { + modules: [ + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + ], + }, + { + modules: [ + { + id: "c", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + ], + }, + ], + }, + }, + ], + }, + }; + } else if (flowPattern == "flow_seq_3_bun") { + return { + path: "flow_seq_3_bun", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 1; }", + }, + }, + { + id: "b", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 2; }", + }, + }, + { + id: "c", + value: { + input_transforms: {}, + language: "bun" as api.RawScript.language, + type: "rawscript", + content: "export function main() { return 3; }", + }, + }, + ], + }, + }; } else { return { path: "2steps", diff --git a/benchmarks/main.ts b/benchmarks/main.ts index 4d8d144cd4..1f681a9c03 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -264,24 +264,6 @@ export async function main({ ); const shutdown_start = Date.now(); - // let zombie_jobs = 0; - // let incorrect_results = 0; - // workers.forEach((worker, i) => { - // const l = (evt: MessageEvent) => { - // if (evt.data.type === "zombie_jobs") { - // zombie_jobs += evt.data.zombie_jobs; - // incorrect_results += evt.data.incorrect_results; - // worker.removeEventListener("message", l); - // workers = workers.filter((w) => w != worker); - // jobsSent[i] = evt.data.jobs_sent; - // worker.terminate(); - // } - // }; - // worker.addEventListener("message", l); - // worker.postMessage( - // Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000 - // ); - // }); workers.forEach((worker, i) => { const l = (evt: MessageEvent) => { if (evt.data.type === "done") { @@ -327,8 +309,6 @@ export async function main({ console.log("time (s + tts):", time); console.log("throughput /s (jobs/time):", sum / time); - // console.log("zombie jobs: ", zombie_jobs); - // console.log("incorrect results: ", incorrect_results); console.log( "queue length:", ( diff --git a/benchmarks/suite_wac.json b/benchmarks/suite_wac.json new file mode 100644 index 0000000000..e677952761 --- /dev/null +++ b/benchmarks/suite_wac.json @@ -0,0 +1,30 @@ +[ + { + "kind": "wac_seq_2", + "jobs": 250 + }, + { + "kind": "wac_par_2", + "jobs": 250 + }, + { + "kind": "wac_seq_3", + "jobs": 200 + }, + { + "kind": "wac_inline_2", + "jobs": 500 + }, + { + "kind": "flow_seq_2_bun", + "jobs": 250 + }, + { + "kind": "flow_par_2_bun", + "jobs": 250 + }, + { + "kind": "flow_seq_3_bun", + "jobs": 200 + } +] diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index cd56d0fb45..ae06fa9779 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -139,96 +139,6 @@ while (cont) { clearInterval(updateStatusInterval); -// const end_time = Date.now() + complete_timeout; - -// let incorrect_results = 0; -// const enc = (s: string) => new TextEncoder().encode(s); - -// let last_queue_length = await getQueueCount(); -// console.log(`waiting for ${last_queue_length} jobs to complete...`); - -// while ( -// outstanding.length > 0 && -// last_queue_length > 0 && -// Date.now() < end_time -// ) { -// try { -// if (!config.hideProgress) { -// await Deno.stdout.write( -// enc( -// "\rwaiting for jobs to complete: outstanding " + -// outstanding.length + -// " - queue" + -// last_queue_length + -// "\n" -// ) -// ); -// } -// last_queue_length = await getQueueCount(); - -// const uuid = outstanding.shift()!; - -// let r: Job; -// try { -// r = await windmill.JobService.getJob({ -// workspace: config.workspace_id, -// id: uuid, -// }); -// } catch (e) { -// console.log("job not found: " + uuid + " " + e.message); -// continue; -// } -// if (r.type == "QueuedJob") { -// outstanding.push(uuid); - -// if (!config.hideProgress) { -// await Deno.stdout.write( -// enc(`uuid: ${uuid}, queue length: ${last_queue_length}\r`) -// ); -// } -// } else { -// r = r as api.CompletedJob; -// try { -// if ( -// ![ -// "httpversion", -// "identity", -// "httpslow", -// "noop", -// "dedicated", -// ].includes(config.scriptPattern) && -// r.result != uuid -// ) { -// console.log( -// "job did not return correct UUID: " + -// r.result + -// " != " + -// uuid + -// "job: \n" + -// JSON.stringify(r, null, 2) -// ); -// incorrect_results++; -// } else { -// // console.log(r.result); -// } -// } catch (e) { -// console.log("error during wait: ", e); -// outstanding.push(uuid); -// } -// } -// } catch (e) { -// console.log("error while waiting for outstanding jobs, sleeing: ", e); -// await sleep(0.5); -// } -// } - -// self.postMessage({ -// type: "zombie_jobs", -// zombie_jobs: outstanding.length, -// incorrect_results, -// jobs_sent: total_spawned, -// }); - self.postMessage({ type: "done", jobs_sent: total_spawned,