#!/usr/bin/env node import { spawn, exec } from "child_process"; import { promisify } from "util"; import * as readline from "readline"; import { WorktreePool, WorktreeInfo } from "./worktree-pool"; import { Logger } from "./logger"; const execAsync = promisify(exec); export interface Config { dbPort: number; serverPort: number; skipBuild: boolean; commitHash: string; worktreePool: WorktreePool; adminPwd: string; onCloudflaredUrl?: (url: string) => void; onCleanup?: () => void; } interface SpawnedResources { dbContainerId: string; dbProcess?: any; backendProcess?: any; cloudflaredProcess?: any; tunnelUrl?: string; worktree?: WorktreeInfo; eeWorktreePath?: string; logger?: Logger; } // No default config needed since commitHash is always required export class EphemeralBackend { private config: Config; private resources: SpawnedResources = { dbContainerId: "" }; getDbPort(): number { return this.config.dbPort; } getServerPort(): number { return this.config.serverPort; } getLogFilePath(): string | undefined { return this.resources.logger?.getLogFilePath(); } constructor(config: Config) { this.config = config; } async spawn(): Promise<{ tunnelUrl: string; }> { try { // Initialize logger for this ephemeral backend this.resources.logger = new Logger(this.config.commitHash, "backend"); this.resources.logger.log("๐Ÿš€ Starting ephemeral backend..."); this.resources.logger.log(`๐Ÿ“Š Database port: ${this.config.dbPort}`); this.resources.logger.log(`๐ŸŒ Server port: ${this.config.serverPort}`); this.resources.logger.log(`๐Ÿ“Œ Commit hash: ${this.config.commitHash}`); await this.startCloudflared(); if (!this.resources.tunnelUrl) throw new Error("Cloudflare tunnel URL not available"); await this.acquireWorktree(); await this.setupEECode(); await this.spawnPostgres(); await this.waitForPostgres(); if (!this.config.skipBuild) { await this.buildBackend(); } else { this.resources.logger?.log( "\nโญ๏ธ Skipping backend build (using existing binary)" ); } await this.startBackend(); // Release the worktree back to the pool now that the backend is running // The binary is already compiled and running, so other spawns can reuse this worktree if (this.resources.worktree) { this.resources.logger?.log( "\nโ™ป๏ธ Releasing worktree back to pool (backend is now running)..." ); await this.config.worktreePool.release(this.resources.worktree.id); // Keep the reference for cleanup but mark it as released this.resources.worktree = undefined; } this.resources.logger?.log("\nโœ… Ephemeral backend is ready!"); this.resources.logger?.log(`๐Ÿ“ Tunnel URL: ${this.resources.tunnelUrl}`); this.resources.logger?.log( `๐Ÿ“„ Log file: ${this.resources.logger.getLogFilePath()}` ); return { tunnelUrl: this.resources.tunnelUrl, }; } catch (error) { this.resources.logger?.error( `โŒ Error spawning ephemeral backend: ${error}` ); throw error; } } private async acquireWorktree(): Promise { this.resources.logger?.log("\n๐Ÿ“‚ Acquiring worktree from pool..."); // Acquire a worktree from the pool this.resources.worktree = await this.config.worktreePool.acquire( this.config.commitHash ); this.resources.logger?.log( `โœ“ Worktree acquired: ${this.resources.worktree.path}` ); } private async setupEECode(): Promise { this.resources.logger?.log("\n๐Ÿ” Setting up Enterprise Edition code..."); if (!this.resources.worktree) { throw new Error("Worktree not acquired"); } const worktreePath = this.resources.worktree.path; const eeRefPath = `${worktreePath}/backend/ee-repo-ref.txt`; const eeWorktreePath = this.config.worktreePool.getEEWorktreePath( this.resources.worktree ); this.resources.eeWorktreePath = eeWorktreePath; // Read the EE commit hash from ee-repo-ref.txt this.resources.logger?.log( ` Reading EE commit reference from ${eeRefPath}` ); let eeCommitHash: string; try { const { stdout } = await execAsync(`cat ${eeRefPath}`); eeCommitHash = stdout.trim(); if (!eeCommitHash) { throw new Error("ee-repo-ref.txt is empty"); } this.resources.logger?.log(` โœ“ EE commit hash: ${eeCommitHash}`); } catch (error) { throw new Error( `Failed to read ee-repo-ref.txt: ${ error instanceof Error ? error.message : String(error) }` ); } // Remove existing EE private folder if it exists (from previous runs) this.resources.logger?.log(` Cleaning up any existing EE repository...`); try { await execAsync(`rm -rf ${eeWorktreePath}`); } catch (error) { // Ignore errors if directory doesn't exist } // Clone the windmill-ee-private repo at the specific commit this.resources.logger?.log( ` Cloning windmill-ee-private at commit ${eeCommitHash}` ); try { await execAsync( `git clone git@github.com:windmill-labs/windmill-ee-private.git ${eeWorktreePath}`, { env: { ...process.env, GIT_SSH_COMMAND: `ssh -i ${process.env.GIT_EE_DEPLOY_KEY_FILE} -o StrictHostKeyChecking=accept-new`, }, } ); this.resources.logger?.log(` โœ“ Repository cloned to ${eeWorktreePath}`); } catch (error) { throw new Error( `Failed to clone windmill-ee-private: ${ error instanceof Error ? error.message : String(error) }` ); } // Checkout the specific commit this.resources.logger?.log(` Checking out commit ${eeCommitHash}`); try { await execAsync(`git checkout ${eeCommitHash}`, { cwd: eeWorktreePath, }); this.resources.logger?.log(` โœ“ Checked out commit ${eeCommitHash}`); } catch (error) { throw new Error( `Failed to checkout EE commit: ${ error instanceof Error ? error.message : String(error) }` ); } // Run the substitute_ee_code.sh script to copy EE files this.resources.logger?.log( ` Running substitute_ee_code.sh to copy EE files` ); try { await execAsync(`./substitute_ee_code.sh --copy -d ${eeWorktreePath}`, { cwd: `${worktreePath}/backend`, }); this.resources.logger?.log(` โœ“ EE code substituted successfully`); } catch (error) { throw new Error( `Failed to substitute EE code: ${ error instanceof Error ? error.message : String(error) }` ); } this.resources.logger?.log("โœ“ Enterprise Edition code setup complete"); } private async spawnPostgres(): Promise { this.resources.logger?.log("\n๐Ÿ˜ Spawning PostgreSQL container..."); this.resources.dbProcess = spawn("docker", [ "run", "--rm", "-p", `${this.config.dbPort}:5432`, "-e", "POSTGRES_PASSWORD=changeme", "-e", "POSTGRES_DB=windmill", "postgres:16", ]); // Capture and log postgres stdout this.resources.dbProcess.stdout.on("data", (data: Buffer) => { const output = data.toString().trim(); if (output) { this.resources.logger?.log(`[postgres] ${output}`); } }); // Capture and log postgres stderr this.resources.dbProcess.stderr.on("data", (data: Buffer) => { const output = data.toString().trim(); if (output) { this.resources.logger?.error(`[postgres] ${output}`); } }); this.resources.dbProcess.on("close", (code: number) => { this.resources.logger?.log(`PostgreSQL process exited with code ${code}`); }); } private async waitForPostgres(): Promise { this.resources.logger?.log("โณ Waiting for PostgreSQL to be ready..."); const maxAttempts = 30; const delayMs = 1000; // Determine the host to connect to // On Linux, we need to use the host's IP or localhost // On macOS/Windows, host.docker.internal works const isLinux = process.platform === "linux"; const dbHost = isLinux ? "172.17.0.1" : "host.docker.internal"; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { await execAsync( `docker run --rm postgres:16 pg_isready -h ${dbHost} -p ${this.config.dbPort} -U postgres` ); this.resources.logger?.log("โœ“ PostgreSQL is ready"); return; } catch (error) { if (attempt === maxAttempts) { throw new Error("PostgreSQL failed to start in time"); } await new Promise((resolve) => setTimeout(resolve, delayMs)); } } } private async buildBackend(): Promise { this.resources.logger?.log( "\n๐Ÿ”จ Building backend (this may take a while)..." ); if (!this.resources.worktree) { throw new Error("Worktree not acquired"); } // Detect OS to use correct deno_core feature const isMacOS = process.platform === "darwin"; const env = { ...process.env, SQLX_OFFLINE: "true" }; const features = [ "enterprise", "enterprise_saml", "stripe", "embedding", "parquet", "prometheus", "openidconnect", "cloud", "jemalloc", "agent_worker_server", "tantivy", "license", "http_trigger", "zip", "oauth2", "kafka", "sqs_trigger", "nats", "otel", "dind", "postgres_trigger", "mqtt_trigger", "gcp_trigger", "websocket", "smtp", "all_languages", "private", isMacOS ? "deno_core_mac" : "deno_core", "mcp", ].join(","); return new Promise((resolve, reject) => { const backendDir = `${this.resources.worktree?.path}/backend`; const buildProcess = spawn( "cargo", ["build", "--features", features, "--release"], { cwd: backendDir, env } ); buildProcess.stdout.on("data", (data) => { const output = data.toString().trim(); if (output) { this.resources.logger?.log(`[build] ${output}`); } }); buildProcess.stderr.on("data", (data) => { const output = data.toString().trim(); if (output) { this.resources.logger?.error(`[build] ${output}`); } }); buildProcess.on("close", (code) => { if (code === 0) { this.resources.logger?.log("โœ“ Backend built successfully"); resolve(); } else { this.resources.logger?.error(`Build failed with code ${code}`); reject(new Error(`Build failed with code ${code}`)); } }); }); } private async startBackend(): Promise { this.resources.logger?.log("\n๐Ÿš€ Starting Windmill backend..."); if (!this.resources.worktree) { throw new Error("Worktree not acquired"); } const env = { // DO NOT PASS the entire process.env to avoid leaking sensitive information ...(process.env.LICENSE_KEY && { LICENSE_KEY: process.env.LICENSE_KEY }), DATABASE_URL: `postgres://postgres:changeme@localhost:${this.config.dbPort}/windmill?sslmode=disable`, PORT: this.config.serverPort.toString(), }; const releaseDir = `${this.resources.worktree.path}/backend/target/release`; // Get sandbox user UID/GID for proper sandboxing const sandboxUid = await this.getSandboxUid(); const sandboxGid = await this.getSandboxGid(); // Use bwrap for sandboxing the windmill process this.resources.backendProcess = spawn( "bwrap", [ ...["--ro-bind", "/", "/"], ...["--bind", "/home/sandbox", "/home/sandbox"], ...["--bind", "/tmp", "/tmp"], ...["--dev", "/dev"], ...["--proc", "/proc"], "--unshare-user", ...["--uid", sandboxUid], ...["--gid", sandboxGid], `${releaseDir}/windmill`, ], { env, } ); // Wait for backend to be ready by watching for "Windmill Enterprise Edition" in output const backendReady = new Promise((resolve, reject) => { // Capture and log backend stdout, watching for ready signal this.resources.backendProcess.stdout.on("data", (data: any) => { const output = data.toString().trim(); const timeout = setTimeout(() => { reject(new Error("Backend failed to start in time")); }, 30000); // 30 seconds timeout if (output) { this.resources.logger?.log(`[backend] ${output}`); if (output.includes("Windmill Enterprise Edition")) { clearTimeout(timeout); resolve(); } } }); }); // Capture and log backend stderr this.resources.backendProcess.stderr.on("data", (data: Buffer) => { const output = data.toString().trim(); if (output) { this.resources.logger?.error(`[backend] ${output}`); } }); this.resources.backendProcess.on("close", (code: number) => { this.resources.logger?.log(`Backend process exited with code ${code}`); }); // Wait for backend to be ready await backendReady; this.resources.logger?.log("โœ“ Backend is running"); // Wait 2 additional seconds to be sure await new Promise((resolve) => setTimeout(resolve, 2000)); // Setup admin password await this.setupAdminPassword(); } private async startCloudflared(): Promise { if (process.env.SKIP_CLOUDFLARED) { this.config.onCloudflaredUrl?.("SKIP_CLOUDFLARED"); return; } this.resources.logger?.log("\n๐ŸŒ Starting Cloudflare tunnel..."); return new Promise((resolve, reject) => { this.resources.cloudflaredProcess = spawn("cloudflared", [ "tunnel", "--url", `http://localhost:${this.config.serverPort}`, "--config", "/dev/null", ]); const rl = readline.createInterface({ input: this.resources.cloudflaredProcess.stdout, }); rl.on("line", (line: string) => { if (line.trim()) { this.resources.logger?.log(`[cloudflared] ${line}`); } }); this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => { const line = data.toString(); if (line.trim()) { this.resources.logger?.error(`[cloudflared] ${line.trim()}`); } const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/); if (match) { this.resources.tunnelUrl = match[1]; this.resources.logger?.log( `โœ“ Tunnel URL: ${this.resources.tunnelUrl}` ); this.config.onCloudflaredUrl?.(this.resources.tunnelUrl); resolve(); } }); this.resources.cloudflaredProcess.on("close", (code: number) => { this.resources.logger?.log( `Cloudflared process exited with code ${code}` ); }); // Timeout if we can't find the URL in 30 seconds setTimeout(() => { if (!this.resources.tunnelUrl) { this.resources.logger?.error( "Failed to extract Cloudflare tunnel URL (timeout)" ); reject(new Error("Failed to extract Cloudflare tunnel URL")); } }, 30000); }); } private async getSandboxUid(): Promise { try { const { stdout } = await execAsync("id -u sandbox"); return stdout.trim(); } catch (error) { this.resources.logger?.error( `Failed to get sandbox UID: ${ error instanceof Error ? error.message : String(error) }` ); throw new Error("Failed to get sandbox user UID"); } } private async getSandboxGid(): Promise { try { const { stdout } = await execAsync("id -g sandbox"); return stdout.trim(); } catch (error) { this.resources.logger?.error( `Failed to get sandbox GID: ${ error instanceof Error ? error.message : String(error) }` ); throw new Error("Failed to get sandbox user GID"); } } private async setupAdminPassword(): Promise { this.resources.logger?.log("\n๐Ÿ”‘ Setting up admin password..."); const baseUrl = `http://localhost:${this.config.serverPort}`; try { // Login with default credentials this.resources.logger?.log(" Logging in with default credentials..."); const loginResponse = await fetch(`${baseUrl}/api/auth/login`, { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ email: "admin@windmill.dev", password: "changeme", }), }); if (!loginResponse.ok) { throw new Error( `Login failed with status ${ loginResponse.status }: ${await loginResponse.text()}` ); } const token = await loginResponse.text(); this.resources.logger?.log(" โœ“ Logged in successfully"); // Set new admin password this.resources.logger?.log(" Setting new admin password..."); const setPasswordResponse = await fetch( `${baseUrl}/api/users/setpassword`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}`, }, body: JSON.stringify({ password: this.config.adminPwd, }), } ); if (!setPasswordResponse.ok) { throw new Error( `Set password failed with status ${ setPasswordResponse.status }: ${await setPasswordResponse.text()}` ); } this.resources.logger?.log("โœ“ Admin password configured successfully"); } catch (error) { this.resources.logger?.error( `Failed to setup admin password: ${ error instanceof Error ? error.message : String(error) }` ); throw error; } } async cleanup(): Promise { this.resources.logger?.log("\n๐Ÿงน Cleaning up resources..."); // Kill backend process if (this.resources.backendProcess) { this.resources.logger?.log(" Stopping backend..."); try { this.resources.backendProcess.kill("SIGTERM"); // Give it a moment to gracefully shutdown await new Promise((resolve) => setTimeout(resolve, 1000)); // Force kill if still running this.resources.backendProcess.kill("SIGKILL"); } catch (error) { // Process might already be dead } } // Kill cloudflared process if (this.resources.cloudflaredProcess) { this.resources.logger?.log(" Stopping cloudflared..."); try { this.resources.cloudflaredProcess.kill("SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 1000)); this.resources.cloudflaredProcess.kill("SIGKILL"); } catch (error) { // Process might already be dead } } // Kill PostgreSQL process if (this.resources.dbProcess) { this.resources.logger?.log(" Stopping PostgreSQL container..."); try { this.resources.dbProcess.kill("SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 1000)); this.resources.dbProcess.kill("SIGKILL"); this.resources.logger?.log(" โœ“ PostgreSQL container stopped"); } catch (error) { console.error(" Failed to stop PostgreSQL container:", error); } } // Remove EE private repository clone if (this.resources.eeWorktreePath) { this.resources.logger?.log(" Removing EE private repository clone..."); try { await execAsync(`rm -rf ${this.resources.eeWorktreePath}`); this.resources.logger?.log(" โœ“ EE private repository clone removed"); } catch (error) { console.error(" Failed to remove EE private repository clone:", error); } } // Release git worktree back to pool (do not delete it) // Note: worktree might already be released if backend started successfully if (this.resources.worktree) { this.resources.logger?.log(" Releasing worktree back to pool..."); try { await this.config.worktreePool.release(this.resources.worktree.id); this.resources.logger?.log(" โœ“ Worktree released for reuse"); } catch (error) { console.error(" Failed to release worktree:", error); } } else { this.resources.logger?.log(" โœ“ Worktree already released"); } this.config.onCleanup?.(); this.resources.logger?.log("โœ… Cleanup complete"); } }