kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fullstack CI Preview (#7665)

Browse Source 
* update cf worker hostname

* set remote_url cookie from param

* ephemeral backends v1

* nit

* Run queue server

* ntis

* timeout

* better db process management

* commit hash and worktree

* nit use map

* nit

* err handling

* Revert "err handling"

This reverts commit 19de00c0c0.

* nits

* auto cleanup

* Ephemeral backend command action

* remove checkout

* checkout ee repo

* nits

* process.env.GIT_EE_DEPLOY_KEY_FILE

* resumeURLs logic

* nit

* use windmill flow for ephemeral backend action

* fixes

* new token

* worktree pools

* Delete GH secret on cleanup

* linux deploy

* nit

* nit

* unhandled promises

* nit

* fix docker bridge IP on linux

* pass cf_frontend_url to wmill flow

* git fetch

* release worktree when binary started

* send error

* logger

* logging

* logging 2

* delete log files periodically

* redirect to raw app with logs

* CORS

* MANAGER_AUTH_TOKEN

* Check organization membership

* nit

* bwrap

* nit

* return timeoutAt in resumeUrl

* nit

* Change password

* nit remove https
This commit is contained in:
Diego Imbert
2026-02-05 18:26:51 +01:00
committed by GitHub
parent 476e6fd4bd
commit 4f2f7356c0
15 changed files with 2226 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
.github/workflows/spawn-ephemeral-backend.yml vendored
View File

@@ -22,6 +22,49 @@ jobs:
contents: read
steps:
- name: Check organization membership
if: github.event_name == 'issue_comment'
id: check-org-member
uses: actions/github-script@v7
with:
script: |
const commenter = context.payload.comment.user.login;
try {
const membership = await github.rest.orgs.checkMembershipForUser({
org: 'windmill-labs',
username: commenter
});
core.setOutput('is_member', 'true');
console.log(`✓ User ${commenter} is a member of windmill-labs`);
} catch (error) {
core.setOutput('is_member', 'false');
console.log(`✗ User ${commenter} is not a member of windmill-labs`);
}
- name: Post unauthorized comment
if: |
github.event_name == 'issue_comment' &&
steps.check-org-member.outputs.is_member == 'false'
uses: actions/github-script@v7
with:
script: |
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `❌ Unauthorized: Only members of the windmill-labs organization can spawn ephemeral backends.`
});
- name: Fail if not org member
if: |
github.event_name == 'issue_comment' &&
steps.check-org-member.outputs.is_member == 'false'
run: |
echo "Error: User is not a member of windmill-labs organization"
exit 1
- name: Get PR details
id: pr-details
uses: actions/github-script@v7
@@ -36,40 +79,80 @@ jobs:
repo: context.repo.repo,
pull_number: prNumber
});
// Get branch name and format it for Cloudflare Pages
// Replace '/' with '-' for the URL
const branchName = pr.data.head.ref;
const formattedBranch = branchName.replace(/\//g, '-');
const cfFrontendUrl = `https://${formattedBranch}.windmill.pages.dev`;
core.setOutput('commit_hash', pr.data.head.sha);
core.setOutput('pr_number', context.issue.number);
core.setOutput('pr_number', prNumber);
core.setOutput('branch_name', branchName);
core.setOutput('cf_frontend_url', cfFrontendUrl);
- name: Trigger Windmill flow
id: trigger-flow
- name: Check manager URL
id: check-manager-url
run: |
RESPONSE=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \
-H "Authorization: Bearer ${{ secrets.WINDMILL_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{
"manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}",
"commit_hash": "${{ steps.pr-details.outputs.commit_hash }}",
"pr_number": ${{ steps.pr-details.outputs.pr_number }}
}')
JOB_UUID=$(echo "$RESPONSE" | jq -r '.id // empty')
if [ -z "$JOB_UUID" ]; then
echo "Failed to get job UUID from response: $RESPONSE"
exit 1
if [ -z "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}" ]; then
echo "manager_url_set=false" >> $GITHUB_OUTPUT
else
echo "manager_url_set=true" >> $GITHUB_OUTPUT
fi
echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT
- name: Post comment with job link
- name: Post error comment if manager not running
if: steps.check-manager-url.outputs.manager_url_set == 'false'
uses: actions/github-script@v7
with:
script: |
const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}';
const jobUrl = `https://app.windmill.dev/run/${jobUuid}?workspace=windmill-labs`;
const prNumber = context.eventName === 'workflow_dispatch'
? Number(context.payload.inputs.pr_number)
: context.issue.number;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `🚀 Ephemeral backend spawning started!\n\nView job progress: ${jobUrl}`
issue_number: prNumber,
body: `❌ Manager URL not set (did you start the ephemeral backend manager?)\n\nThe ephemeral backend manager needs to be running to spawn backends. Please start the manager first.`
});
- name: Fail if manager not running
if: steps.check-manager-url.outputs.manager_url_set == 'false'
run: |
echo "Error: EPHEMERAL_BACKEND_QUEUE_URL secret is not set"
exit 1
- name: Trigger Windmill flow
if: steps.check-manager-url.outputs.manager_url_set == 'true'
id: trigger-flow
run: |
JOB_UUID=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \
-H "Authorization: Bearer ${{ secrets.WINDMILL_RUN_FLOW_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{
"manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}",
"commit_hash": "${{ steps.pr-details.outputs.commit_hash }}",
"pr_number": ${{ steps.pr-details.outputs.pr_number }},
"cf_frontend_url": "${{ steps.pr-details.outputs.cf_frontend_url }}"
}' | tr -d '"')
echo "Job UUID: $JOB_UUID"
echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT
- name: Post comment with job link
if: steps.check-manager-url.outputs.manager_url_set == 'true'
uses: actions/github-script@v7
with:
script: |
const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}';
const appUrl = `https://app.windmill.dev/public/windmill-labs/a106bad0256c1dfa7a4f9279c42b1a4b#${jobUuid}`;
const prNumber = context.eventName === 'workflow_dispatch'
? Number(context.payload.inputs.pr_number)
: context.issue.number;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: `🚀 Spawning new ephemeral backend!\n\n${appUrl}`
});

15
ephemeral-backends/.env.template Normal file
View File

@@ -0,0 +1,15 @@
# Windmill Ephemeral Backend Manager - Environment Variables
# Copy this file to /etc/windmill-ephemeral-manager/.env and fill in the values
# Manager Authentication Token
# Required for protecting the manager endpoints from unauthorized access
# Generate a secure random token: openssl rand -hex 32
MANAGER_AUTH_TOKEN=your-secure-random-token-here
# GitHub Personal Access Token with 'secrets' scope (Write access)
GITHUB_TOKEN=github_pat_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
# Path to the SSH deploy key for accessing windmill-ee-private repository
# This should be a read-only deploy key
# The file should be owned by the 'sandbox' user with 600 permissions
GIT_EE_DEPLOY_KEY_FILE=/home/sandbox/.ssh/windmill_ee_deploy_key

5
ephemeral-backends/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
node_modules/
dist/
*.log
.DS_Store
.env

35
ephemeral-backends/DEPLOYMENT.md Normal file
View File

@@ -0,0 +1,35 @@
# Deploying the Ephemeral Backend Manager
This guide explains how to deploy the Windmill Ephemeral Backend Manager as a systemd service on a Linux machine.
## Prerequisites
Before running the installation script, ensure you have:
1. **Linux machine** with systemd (Ubuntu 20.04+, Debian 11+, etc.)
2. **User `sandbox`** created with appropriate permissions
3. **Repository cloned** to `/home/sandbox/ephemeral-backend/windmill`
4. **Required tools installed**:
- Git
- Docker (with `sandbox` user having access)
- Bun (installed for the `sandbox` user)
- Rust/Cargo (for building backends)
- Bubblewrap (will be installed by the script if missing)
- Cloudflared (will be installed by the script if missing)
## Architecture
The service runs as user `sandbox` and:
- Listens on port 8001 for HTTP requests
- Creates a Cloudflare tunnel for external access
- Updates GitHub Actions secrets with the tunnel URL
- Manages a pool of git worktrees for ephemeral backends
- Spawns ephemeral backends on-demand
- Automatically cleans up resources on shutdown
The worktree pool enables:
- Fast incremental compilation (reuses target directories)
- Efficient resource usage (no constant creation/deletion)
- Automatic discovery of existing worktrees on restart

167
ephemeral-backends/install.sh Executable file
View File

@@ -0,0 +1,167 @@
#!/bin/bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo "🎛️ Windmill Ephemeral Backend Manager - Installation Script"
echo "=========================================================="
echo ""
# Check if running as root
if [ "$EUID" -ne 0 ]; then
echo -e "${RED}❌ This script must be run as root (use sudo)${NC}"
exit 1
fi
# Get the actual user who ran sudo (if applicable)
ACTUAL_USER="${SUDO_USER:-$(whoami)}"
# Configuration
SERVICE_USER="sandbox"
SERVICE_NAME="windmill-ephemeral-manager"
REPO_DIR="/home/$SERVICE_USER/ephemeral-backend/windmill"
ENV_DIR="/etc/$SERVICE_NAME"
ENV_FILE="$ENV_DIR/.env"
echo "📋 Configuration:"
echo " Service user: $SERVICE_USER"
echo " Repository directory: $REPO_DIR"
echo " Environment file: $ENV_FILE"
echo ""
# Check if sandbox user exists
if ! id "$SERVICE_USER" &>/dev/null; then
echo -e "${RED}❌ User '$SERVICE_USER' does not exist${NC}"
echo " Create the user first: sudo adduser --system --group $SERVICE_USER"
exit 1
fi
# Check if repository directory exists
if [ ! -d "$REPO_DIR" ]; then
echo -e "${RED}❌ Repository directory does not exist: $REPO_DIR${NC}"
echo " Clone the repository first as user $SERVICE_USER"
exit 1
fi
# Check if bun is installed for the service user
if ! sudo -u "$SERVICE_USER" bash -c "command -v bun" &>/dev/null; then
echo -e "${RED}❌ Bun is not installed for user $SERVICE_USER${NC}"
echo " Install bun: curl -fsSL https://bun.sh/install | bash"
exit 1
fi
# Check if bubblewrap is installed
if ! command -v bwrap &>/dev/null; then
echo -e "${YELLOW}⚠️ bubblewrap is not installed${NC}"
echo " Installing bubblewrap..."
apt-get update -qq
apt-get install -y -qq bubblewrap
echo -e "${GREEN}✓ bubblewrap installed${NC}"
fi
# Check if cloudflared is installed
if ! command -v cloudflared &>/dev/null; then
echo -e "${YELLOW}⚠️ cloudflared is not installed${NC}"
echo " Installing cloudflared..."
# Detect architecture
ARCH=$(uname -m)
if [ "$ARCH" = "x86_64" ]; then
wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb
dpkg -i cloudflared-linux-amd64.deb
rm cloudflared-linux-amd64.deb
elif [ "$ARCH" = "aarch64" ]; then
wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-arm64.deb
dpkg -i cloudflared-linux-arm64.deb
rm cloudflared-linux-arm64.deb
else
echo -e "${RED}❌ Unsupported architecture: $ARCH${NC}"
exit 1
fi
echo -e "${GREEN}✓ cloudflared installed${NC}"
fi
# Check if docker is installed and service user has access
if ! command -v docker &>/dev/null; then
echo -e "${RED}❌ Docker is not installed${NC}"
echo " Install docker: https://docs.docker.com/engine/install/"
exit 1
fi
if ! sudo -u "$SERVICE_USER" docker ps &>/dev/null; then
echo -e "${YELLOW}⚠️ User $SERVICE_USER cannot access Docker${NC}"
echo " Adding $SERVICE_USER to docker group..."
usermod -aG docker "$SERVICE_USER"
echo -e "${GREEN}✓ User added to docker group (may require logout/login)${NC}"
fi
# Create environment directory
echo ""
echo "📁 Setting up environment..."
mkdir -p "$ENV_DIR"
chmod 755 "$ENV_DIR"
# Copy or create environment file
if [ ! -f "$ENV_FILE" ]; then
if [ -f "$REPO_DIR/ephemeral-backends/.env.template" ]; then
cp "$REPO_DIR/ephemeral-backends/.env.template" "$ENV_FILE"
echo -e "${YELLOW}⚠️ Environment file created from template: $ENV_FILE${NC}"
echo " ${RED}IMPORTANT: Edit this file and fill in the required values!${NC}"
else
echo -e "${RED}❌ Template file not found: $REPO_DIR/ephemeral-backends/.env.template${NC}"
exit 1
fi
else
echo -e "${GREEN}✓ Environment file already exists: $ENV_FILE${NC}"
fi
chmod 600 "$ENV_FILE"
# Install systemd service
echo ""
echo "🔧 Installing systemd service..."
cp "$REPO_DIR/ephemeral-backends/windmill-ephemeral-manager.service" "/etc/systemd/system/$SERVICE_NAME.service"
chmod 644 "/etc/systemd/system/$SERVICE_NAME.service"
echo -e "${GREEN}✓ Service file installed${NC}"
# Reload systemd
echo ""
echo "🔄 Reloading systemd..."
systemctl daemon-reload
echo -e "${GREEN}✓ Systemd reloaded${NC}"
# Enable service
echo ""
echo "✅ Enabling service..."
systemctl enable "$SERVICE_NAME.service"
echo -e "${GREEN}✓ Service enabled (will start on boot)${NC}"
echo ""
echo "=========================================================="
echo -e "${GREEN}✅ Installation complete!${NC}"
echo ""
echo "📝 Next steps:"
echo " 1. Edit the environment file: sudo nano $ENV_FILE"
echo " 2. Fill in the required values:"
echo " - MANAGER_AUTH_TOKEN (generate with: openssl rand -hex 32)"
echo " - GITHUB_TOKEN (GitHub personal access token)"
echo " - GIT_EE_DEPLOY_KEY_FILE (path to SSH deploy key)"
echo " 3. Ensure the SSH deploy key exists and is readable by $SERVICE_USER"
echo " 4. Add MANAGER_AUTH_TOKEN to GitHub repository secrets"
echo " 5. Start the service: sudo systemctl start $SERVICE_NAME"
echo " 6. Check status: sudo systemctl status $SERVICE_NAME"
echo " 7. View logs: sudo journalctl -u $SERVICE_NAME -f"
echo ""
echo "🔍 Useful commands:"
echo " sudo systemctl start $SERVICE_NAME # Start the service"
echo " sudo systemctl stop $SERVICE_NAME # Stop the service"
echo " sudo systemctl restart $SERVICE_NAME # Restart the service"
echo " sudo systemctl status $SERVICE_NAME # Check service status"
echo " sudo journalctl -u $SERVICE_NAME -f # Follow logs"
echo " sudo journalctl -u $SERVICE_NAME -n 100 # Last 100 log lines"
echo ""

131
ephemeral-backends/logger.ts Normal file
View File

@@ -0,0 +1,131 @@
import {
appendFileSync,
mkdirSync,
unlinkSync,
readdirSync,
statSync,
} from "fs";
import path from "path";
const LOG_DIR = "/tmp/windmill-ephemeral-logs";
const LOG_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
export class Logger {
private logFilePath: string;
private prefix: string;
constructor(identifier: string, prefix: string = "") {
// Sanitize identifier to prevent path traversal
const sanitizedId = identifier.replace(/[^a-zA-Z0-9-_]/g, "_");
// Ensure log directory exists
try {
mkdirSync(LOG_DIR, { recursive: true });
} catch (error) {
// Directory might already exist
}
this.logFilePath = path.join(LOG_DIR, `${sanitizedId}.log`);
this.prefix = prefix;
// Write initial log entry
const startTime = new Date().toISOString();
this.log(`=== Log started at ${startTime} ===`);
}
private formatMessage(level: string, message: string): string {
const timestamp = new Date().toISOString();
const prefixStr = this.prefix ? `[${this.prefix}] ` : "";
return `${timestamp} ${level} ${prefixStr}${message}\n`;
}
log(message: string): void {
const formatted = this.formatMessage("stdin ", message);
try {
appendFileSync(this.logFilePath, formatted);
} catch (error) {
// Fallback to console if file write fails
console.error("Failed to write to log file:", error);
console.log(formatted);
}
// Also log to stdout for systemd journal
process.stdout.write(formatted);
}
error(message: string): void {
const formatted = this.formatMessage("stderr", message);
try {
appendFileSync(this.logFilePath, formatted);
} catch (error) {
console.error("Failed to write to log file:", error);
console.error(formatted);
}
// Also log to stderr for systemd journal
process.stderr.write(formatted);
}
warn(message: string): void {
const formatted = this.formatMessage("warn ", message);
try {
appendFileSync(this.logFilePath, formatted);
} catch (error) {
console.error("Failed to write to log file:", error);
console.warn(formatted);
}
process.stdout.write(formatted);
}
getLogFilePath(): string {
return this.logFilePath;
}
static getLogFilePathForCommit(commitHash: string): string {
// Sanitize commit hash to prevent path traversal
const sanitizedHash = commitHash.replace(/[^a-f0-9]/g, "");
if (sanitizedHash.length < 7 || sanitizedHash.length > 40) {
throw new Error("Invalid commit hash");
}
return path.join(LOG_DIR, `${sanitizedHash}.log`);
}
/**
* Clean up old log files (older than 24 hours)
*/
static cleanupOldLogs(): void {
try {
const files = readdirSync(LOG_DIR);
const now = Date.now();
let deletedCount = 0;
for (const file of files) {
if (!file.endsWith(".log")) continue;
const filePath = path.join(LOG_DIR, file);
try {
const stats = statSync(filePath);
const ageMs = now - stats.mtimeMs;
if (ageMs > LOG_RETENTION_MS) {
unlinkSync(filePath);
deletedCount++;
console.log(
`Deleted old log file: ${file} (age: ${Math.floor(
ageMs / 1000 / 60 / 60
)}h)`
);
}
} catch (error) {
// Skip files we can't stat or delete
console.error(`Failed to process log file ${file}:`, error);
}
}
if (deletedCount > 0) {
console.log(
`Cleanup complete: deleted ${deletedCount} old log file(s)`
);
}
} catch (error) {
console.error("Failed to cleanup old logs:", error);
}
}
}

687
ephemeral-backends/manager.ts Normal file
View File

@@ -0,0 +1,687 @@
#!/usr/bin/env node
import { spawn } from "child_process";
import * as readline from "readline";
import sodium from "libsodium-wrappers-sumo";
import { EphemeralBackend } from "./spawn";
import { WorktreePool } from "./worktree-pool";
import { Logger } from "./logger";
import { readFileSync, existsSync } from "fs";
process.on("unhandledRejection", (err) => {
console.error("UNHANDLED PROMISE:", err);
});
process.on("uncaughtException", (err) => {
console.error("UNCAUGHT EXCEPTION:", err);
});
const githubToken = process.env.GITHUB_TOKEN;
if (!githubToken) {
console.log("⚠️ GITHUB_TOKEN environment variable not set");
console.log("\n📝 Set a GitHub token with 'secrets' scope:");
console.log(" export GITHUB_TOKEN=github_pat_...");
process.exit(1);
}
const managerAuthToken = process.env.MANAGER_AUTH_TOKEN;
if (!managerAuthToken) {
console.log("⚠️ MANAGER_AUTH_TOKEN environment variable not set");
console.log("\n📝 Set a secure random token for API authentication:");
console.log(" export MANAGER_AUTH_TOKEN=$(openssl rand -hex 32)");
process.exit(1);
}
if (!process.env.GIT_EE_DEPLOY_KEY_FILE) {
console.log("⚠️ GIT_EE_DEPLOY_KEY_FILE environment variable not set");
console.log("\n📝 Set a read-only SSH deploy key:");
console.log(" export GIT_EE_DEPLOY_KEY_FILE=/home/...");
process.exit(1);
}
const MANAGER_PORT = 8001;
const BACKEND_TIMEOUT_MS = 120 * 60 * 1000; // 2 hours in milliseconds
interface BackendInfo {
backend: EphemeralBackend;
timeoutId: NodeJS.Timeout;
createdAt: Date;
}
interface ManagerResources {
cloudflaredProcess?: any;
tunnelUrl?: string;
ephemeralBackends: Map<string, BackendInfo>;
worktreePool?: WorktreePool;
cleanupInterval?: NodeJS.Timeout;
}
class EphemeralBackendManager {
private resources: ManagerResources = {
ephemeralBackends: new Map(),
};
private server?: any;
async start(): Promise<void> {
// Setup cleanup handlers early
process.on("SIGINT", () => this.cleanup());
process.on("SIGTERM", () => this.cleanup());
try {
console.log("🎛️ Starting Ephemeral Backend Manager...");
console.log(`📊 Manager port: ${MANAGER_PORT}`);
// Initialize the worktree pool
this.resources.worktreePool = new WorktreePool();
await this.resources.worktreePool.initialize();
// Set up periodic log cleanup (every 6 hours)
this.resources.cleanupInterval = setInterval(() => {
console.log("\n🧹 Running periodic log cleanup...");
Logger.cleanupOldLogs();
}, 6 * 60 * 60 * 1000);
await this.startHttpServer();
if (!process.env.SKIP_CLOUDFLARED) await this.startCloudflared();
if (!process.env.SKIP_SET_GH_SECRET) await this.updateGitHubSecret();
console.log("\n✅ Manager is ready!");
console.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`);
console.log("\n💡 Press Ctrl+C to stop...");
// Keep the process running indefinitely
await new Promise(() => {}); // Never resolves
} catch (error) {
console.error("❌ Error starting manager:", error);
await this.cleanup();
process.exit(1);
}
}
private async startHttpServer(): Promise<void> {
const self = this;
console.log("\n🌐 Starting HTTP server...");
return new Promise((resolve) => {
// Use Bun's built-in HTTP server
this.server = Bun.serve({
port: MANAGER_PORT,
idleTimeout: 30,
async fetch(req) {
const url = new URL(req.url);
// CORS headers for app.windmill.dev
const origin = req.headers.get("origin");
const corsHeaders: Record<string, string> = {};
if (origin === "https://app.windmill.dev") {
corsHeaders["Access-Control-Allow-Origin"] = origin;
corsHeaders["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS";
corsHeaders["Access-Control-Allow-Headers"] =
"Content-Type, Authorization";
corsHeaders["Access-Control-Max-Age"] = "86400";
}
// Handle preflight requests
if (req.method === "OPTIONS") {
return new Response(null, {
status: 204,
headers: corsHeaders,
});
}
// Authentication check function
const checkAuth = (): boolean => {
const authHeader = req.headers.get("authorization");
return authHeader === `Bearer ${managerAuthToken}`;
};
// Unauthorized response helper
const unauthorizedResponse = () => {
return new Response(
JSON.stringify({
error: "Unauthorized",
message: "Valid Bearer token required in Authorization header",
}),
{
status: 401,
headers: {
...corsHeaders,
"Content-Type": "application/json",
"WWW-Authenticate": 'Bearer realm="Manager API"',
},
}
);
};
// Health check endpoint
if (url.pathname === "/health") {
return new Response(
JSON.stringify({
status: "ok",
timestamp: new Date().toISOString(),
}),
{
headers: { ...corsHeaders, "Content-Type": "application/json" },
}
);
}
// Status endpoint - shows all running backends and worktree pool stats
if (url.pathname === "/status") {
if (!checkAuth()) {
return unauthorizedResponse();
}
const backends = Array.from(
self.resources.ephemeralBackends.entries()
).map(([commitHash, backendInfo]) => {
const now = new Date();
const timeoutAt = new Date(
backendInfo.createdAt.getTime() + BACKEND_TIMEOUT_MS
);
const timeRemainingMs = timeoutAt.getTime() - now.getTime();
const timeRemainingMinutes = Math.floor(
timeRemainingMs / 1000 / 60
);
return {
commitHash,
shortHash: commitHash.substring(0, 8),
createdAt: backendInfo.createdAt.toISOString(),
timeoutAt: timeoutAt.toISOString(),
timeRemainingMinutes,
serverPort: backendInfo.backend.getServerPort(),
dbPort: backendInfo.backend.getDbPort(),
};
});
const worktreePoolStats =
self.resources.worktreePool?.getStats() || {
total: 0,
inUse: 0,
available: 0,
};
return new Response(
JSON.stringify({
activeBackends: backends.length,
backends,
worktreePool: worktreePoolStats,
timestamp: new Date().toISOString(),
}),
{
headers: { ...corsHeaders, "Content-Type": "application/json" },
}
);
}
// Match /logs/{commit_hash} - serve log files
const logsMatch = url.pathname.match(/^\/logs\/([a-f0-9]+)$/);
if (logsMatch && req.method === "GET") {
if (!checkAuth()) {
return unauthorizedResponse();
}
const commitHash = logsMatch[1];
// Validate commit hash format (7-40 hex characters)
if (commitHash.length < 7 || commitHash.length > 40) {
return new Response("Invalid commit hash", { status: 400 });
}
try {
// Use the Logger class's secure path resolution
const logFilePath = Logger.getLogFilePathForCommit(commitHash);
// Check if file exists
if (!existsSync(logFilePath)) {
return new Response("Log file not found", {
status: 404,
headers: corsHeaders,
});
}
// Read the log file
const logContent = readFileSync(logFilePath, "utf-8");
return new Response(logContent, {
headers: {
...corsHeaders,
"Content-Type": "text/plain; charset=utf-8",
"X-Commit-Hash": commitHash,
},
});
} catch (error: any) {
console.error(`Error reading log file for ${commitHash}:`, error);
return new Response(`Error reading log file: ${error.message}`, {
status: 500,
headers: corsHeaders,
});
}
}
// Match /spawn/{commit_hash}
const spawnMatch = url.pathname.match(/^\/spawn\/([a-f0-9]+)$/);
if (spawnMatch && req.method === "POST") {
if (!checkAuth()) {
return unauthorizedResponse();
}
let body = await req.json();
if (typeof body !== "object")
return new Response("Invalid JSON body", { status: 400 });
let resumeUrl = body?.resume_url;
if (typeof resumeUrl !== "string")
return new Response("Invalid resume_url", { status: 400 });
let cancelUrl = body?.cancel_url;
if (typeof cancelUrl !== "string")
return new Response("Invalid cancel_url", { status: 400 });
const commitHash = spawnMatch[1];
console.log(
`\n🔹 Received request to spawn ephemeral backend for commit: ${commitHash}`
);
if (self.resources.ephemeralBackends.has(commitHash)) {
throw new Error(`Backend ${commitHash} is already running`);
}
if (!self.resources.worktreePool) {
throw new Error("Worktree pool not initialized");
}
const tunnelUrl = await new Promise<string>((res, err) => {
const adminRandomPwd = Math.random()
.toString(36)
.substring(2, 15);
const ephemeralBackend = new EphemeralBackend({
dbPort: self.findFreeDbPorts(),
serverPort: self.findFreeServerPorts(),
skipBuild: !!process.env.SKIP_BACKEND_BUILD,
commitHash: commitHash,
worktreePool: self.resources.worktreePool!,
adminPwd: adminRandomPwd,
onCloudflaredUrl: (url) => (res(url), clearTimeout(timeout)),
onCleanup: () => {
const backendInfo =
self.resources.ephemeralBackends.get(commitHash);
if (backendInfo) {
clearTimeout(backendInfo.timeoutId);
self.resources.ephemeralBackends.delete(commitHash);
}
},
});
function onError(e: any) {
ephemeralBackend.cleanup().catch(() => {
console.error(
`Failed to cleanup backend for commit ${commitHash}`
);
});
clearTimeout(timeout);
fetch(resumeUrl, {
// Cancel URL doesn't show any relevant info, use resume URL
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
status: "error",
commitHash,
error: e.message,
}),
}).catch((e) => {
console.error(
`Failed to notify cancel URL for commit ${commitHash}:`,
e
);
});
}
const timeout = setTimeout(() => {
onError(new Error("Timeout waiting for backend URL"));
}, 20000);
try {
ephemeralBackend
.spawn()
.then(({ tunnelUrl }) => {
if (resumeUrl) {
fetch(resumeUrl, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
status: "ready",
timeoutAt:
(self.resources.ephemeralBackends
.get(commitHash)
?.createdAt.getTime() ?? 0) + BACKEND_TIMEOUT_MS,
commitHash,
adminPassword: adminRandomPwd,
tunnelUrl,
}),
}).catch((e) => {
onError(e);
});
}
})
.catch((e) => onError(e));
// Set up 1-hour timeout for automatic cleanup
const cleanupTimeoutId = setTimeout(async () => {
console.log(
`\n⏰ Backend ${commitHash} has reached 1-hour timeout, cleaning up...`
);
try {
await ephemeralBackend.cleanup();
self.resources.ephemeralBackends.delete(commitHash);
console.log(
`✓ Backend ${commitHash} cleaned up after timeout`
);
} catch (error) {
console.error(
`❌ Failed to cleanup backend ${commitHash} after timeout:`,
error
);
}
}, BACKEND_TIMEOUT_MS);
self.resources.ephemeralBackends.set(commitHash, {
backend: ephemeralBackend,
timeoutId: cleanupTimeoutId,
createdAt: new Date(),
});
} catch (e) {
onError(e);
}
});
return new Response(
JSON.stringify({
tunnelUrl,
timestamp: new Date().toISOString(),
}),
{
headers: { ...corsHeaders, "Content-Type": "application/json" },
status: 202,
}
);
}
// Default 404
return new Response("Not Found", {
status: 404,
headers: corsHeaders,
});
},
});
console.log(`✓ HTTP server listening on port ${MANAGER_PORT}`);
resolve();
});
}
private async startCloudflared(): Promise<void> {
console.log("\n🌐 Starting Cloudflare tunnel for manager...");
return new Promise((resolve, reject) => {
this.resources.cloudflaredProcess = spawn("cloudflared", [
"tunnel",
"--url",
`http://localhost:${MANAGER_PORT}`,
"--config",
"/dev/null",
]);
const rl = readline.createInterface({
input: this.resources.cloudflaredProcess.stdout,
});
rl.on("line", (line: string) => {
console.log(`[cloudflared] ${line}`);
});
this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => {
process.stderr.write(`[cloudflared] ${data}`);
const line = data.toString();
const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/);
if (match) {
this.resources.tunnelUrl = match[1];
console.log(`✓ Tunnel URL extracted: ${this.resources.tunnelUrl}`);
resolve();
}
});
this.resources.cloudflaredProcess.on("close", (code: number) => {
console.log(`Cloudflared process exited with code ${code}`);
});
// Timeout if we can't find the URL in 30 seconds
setTimeout(() => {
if (!this.resources.tunnelUrl) {
reject(new Error("Failed to extract Cloudflare tunnel URL"));
}
}, 30000);
});
}
private async updateGitHubSecret(): Promise<void> {
console.log("\n🔐 Updating GitHub Actions secret...");
if (!this.resources.tunnelUrl) {
console.error("❌ No tunnel URL available to update secret");
return;
}
const fullUrl = `https://${this.resources.tunnelUrl}`;
const repo = "windmill-labs/windmill";
const secretName = "EPHEMERAL_BACKEND_QUEUE_URL";
try {
// First, get the repository public key for encrypting the secret
console.log(" Fetching repository public key...");
const keyResponse = await fetch(
`https://api.github.com/repos/${repo}/actions/secrets/public-key`,
{
headers: {
Authorization: `Bearer ${githubToken}`,
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
}
);
if (!keyResponse.ok) {
throw new Error(
`Failed to fetch public key: ${keyResponse.statusText}`
);
}
const { key, key_id } = await keyResponse.json();
// Encrypt the secret using libsodium (via tweetnacl for Bun compatibility)
console.log(" Encrypting secret value...");
await sodium.ready;
const messageBytes = new TextEncoder().encode(fullUrl);
const keyBytes = sodium.from_base64(key, sodium.base64_variants.ORIGINAL);
const encryptedBytes = sodium.crypto_box_seal(messageBytes, keyBytes);
const encryptedValue = sodium.to_base64(
encryptedBytes,
sodium.base64_variants.ORIGINAL
);
// Update the secret
console.log(" Updating secret...");
const updateResponse = await fetch(
`https://api.github.com/repos/${repo}/actions/secrets/${secretName}`,
{
method: "PUT",
headers: {
Authorization: `Bearer ${githubToken}`,
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"Content-Type": "application/json",
},
body: JSON.stringify({
encrypted_value: encryptedValue,
key_id: key_id,
}),
}
);
if (!updateResponse.ok) {
const errorText = await updateResponse.text();
throw new Error(
`Failed to update secret: ${updateResponse.statusText} - ${errorText}`
);
}
console.log(`✓ GitHub secret updated successfully!`);
console.log(` Repository: ${repo}`);
console.log(` Secret: ${secretName}`);
console.log(` Value: ${fullUrl}`);
} catch (error: any) {
console.error("❌ Failed to update GitHub secret:", error.message);
}
}
private async deleteGitHubSecret(): Promise<void> {
const repo = "windmill-labs/windmill";
const secretName = "EPHEMERAL_BACKEND_QUEUE_URL";
try {
console.log(" Deleting GitHub Actions secret...");
const deleteResponse = await fetch(
`https://api.github.com/repos/${repo}/actions/secrets/${secretName}`,
{
method: "DELETE",
headers: {
Authorization: `Bearer ${githubToken}`,
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
}
);
if (!deleteResponse.ok) {
// 404 is acceptable - secret might not exist
if (deleteResponse.status === 404) {
console.log(
` ✓ Secret ${secretName} does not exist (already deleted)`
);
return;
}
const errorText = await deleteResponse.text();
throw new Error(
`Failed to delete secret: ${deleteResponse.statusText} - ${errorText}`
);
}
console.log(`✓ GitHub secret deleted successfully!`);
console.log(` Repository: ${repo}`);
console.log(` Secret: ${secretName}`);
} catch (error: any) {
console.error("❌ Failed to delete GitHub secret:", error.message);
}
}
isCleaningUp: boolean = false;
private async cleanup(): Promise<void> {
if (this.isCleaningUp) return;
this.isCleaningUp = true;
console.log("\n🧹 Cleaning up manager resources...");
// Stop periodic cleanup interval
if (this.resources.cleanupInterval) {
console.log(" Stopping periodic log cleanup...");
clearInterval(this.resources.cleanupInterval);
this.resources.cleanupInterval = undefined;
}
// Delete GitHub secret
if (!process.env.SKIP_SET_GH_SECRET) {
console.log(" Deleting GitHub Actions secret...");
try {
await this.deleteGitHubSecret();
} catch (error) {
console.error(" Failed to delete GitHub secret:", error);
}
}
// Stop HTTP server
if (this.server) {
console.log(" Stopping HTTP server...");
try {
this.server.stop();
} catch (error) {
console.error(" Failed to stop HTTP server:", error);
}
}
// Kill cloudflared process
if (this.resources.cloudflaredProcess) {
console.log(" Stopping cloudflared...");
try {
this.resources.cloudflaredProcess.kill("SIGTERM");
await new Promise((resolve) => setTimeout(resolve, 1000));
this.resources.cloudflaredProcess.kill("SIGKILL");
} catch (error) {
// Process might already be dead
}
}
for (const [commitHash, backendInfo] of this.resources.ephemeralBackends) {
const hash = commitHash.substring(0, 8);
console.log(
` Cleaning up ephemeral backend ${hash} on port ${backendInfo.backend.getServerPort()}...`
);
try {
clearTimeout(backendInfo.timeoutId); // Clear the timeout before cleanup
await backendInfo.backend.cleanup();
} catch (error) {
console.error(
` Failed to clean up backend ${hash} on port ${backendInfo.backend.getServerPort()}:`,
error
);
}
}
console.log("✅ Cleanup complete");
process.exit(0);
}
private findFreeDbPorts(): number {
const minPort = 5433;
for (let port = minPort; port < minPort + 100; port++) {
if (
![...this.resources.ephemeralBackends.values()].some(
(backendInfo) => port === backendInfo.backend.getDbPort()
)
) {
return port;
}
}
throw new Error("No free DB ports available");
}
private findFreeServerPorts(): number {
const minPort = 8002;
for (let port = minPort; port < minPort + 100; port++) {
if (
![...this.resources.ephemeralBackends.values()].some(
(backendInfo) => port === backendInfo.backend.getServerPort()
)
) {
return port;
}
}
throw new Error("No free server ports available");
}
}
// Main execution
async function main() {
const manager = new EphemeralBackendManager();
await manager.start();
}
main().catch((error) => {
console.error("Fatal error:", error);
process.exit(1);
});

88
ephemeral-backends/package-lock.json generated Normal file
View File

@@ -0,0 +1,88 @@
{
"name": "windmill-ephemeral-backends",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-ephemeral-backends",
"version": "1.0.0",
"dependencies": {
"@types/libsodium-wrappers-sumo": "^0.7.8",
"libsodium-wrappers-sumo": "^0.8.1"
},
"devDependencies": {
"@types/bun": "^1.3.6",
"@types/node": "^20.0.0",
"bun-types": "^1.0.0"
}
},
"node_modules/@types/bun": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.3.6.tgz",
"integrity": "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA==",
"dev": true,
"license": "MIT",
"dependencies": {
"bun-types": "1.3.6"
}
},
"node_modules/@types/libsodium-wrappers": {
"version": "0.7.14",
"resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz",
"integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==",
"license": "MIT"
},
"node_modules/@types/libsodium-wrappers-sumo": {
"version": "0.7.8",
"resolved": "https://registry.npmjs.org/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.8.tgz",
"integrity": "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw==",
"license": "MIT",
"dependencies": {
"@types/libsodium-wrappers": "*"
}
},
"node_modules/@types/node": {
"version": "20.19.30",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.30.tgz",
"integrity": "sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/bun-types": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.3.6.tgz",
"integrity": "sha512-OlFwHcnNV99r//9v5IIOgQ9Uk37gZqrNMCcqEaExdkVq3Avwqok1bJFmvGMCkCE0FqzdY8VMOZpfpR3lwI+CsQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/libsodium-sumo": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.8.1.tgz",
"integrity": "sha512-q8EhpXKjbzWsLZEY3qOym5nA7emtTe/izt6ziSUbUfE8hk2eDP+7Hy0G0bum65JHDChkdtrImNZTR7N5WuiIXg==",
"license": "ISC"
},
"node_modules/libsodium-wrappers-sumo": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.1.tgz",
"integrity": "sha512-xcX1+mQbdwko50AsZWn5r4crZO5H5bHh9ceMHsjBxPyrMn0zFHycms2jcHvXzk71nh4V0rQGCiOjB2ckshd/ew==",
"license": "ISC",
"dependencies": {
"libsodium-sumo": "^0.8.0"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
}
}
}

19
ephemeral-backends/package.json Normal file
View File

@@ -0,0 +1,19 @@
{
"name": "windmill-ephemeral-backends",
"version": "1.0.0",
"description": "Ephemeral backend spawner for Windmill",
"type": "module",
"scripts": {
"start": "bun run spawn.ts",
"manager": "bun run manager.ts"
},
"dependencies": {
"@types/libsodium-wrappers-sumo": "^0.7.8",
"libsodium-wrappers-sumo": "^0.8.1"
},
"devDependencies": {
"@types/bun": "^1.3.6",
"@types/node": "^20.0.0",
"bun-types": "^1.0.0"
}
}

670
ephemeral-backends/spawn.ts Normal file
View File

@@ -0,0 +1,670 @@
#!/usr/bin/env node
import { spawn, exec } from "child_process";
import { promisify } from "util";
import * as readline from "readline";
import { WorktreePool, WorktreeInfo } from "./worktree-pool";
import { Logger } from "./logger";
const execAsync = promisify(exec);
export interface Config {
dbPort: number;
serverPort: number;
skipBuild: boolean;
commitHash: string;
worktreePool: WorktreePool;
adminPwd: string;
onCloudflaredUrl?: (url: string) => void;
onCleanup?: () => void;
}
interface SpawnedResources {
dbContainerId: string;
dbProcess?: any;
backendProcess?: any;
cloudflaredProcess?: any;
tunnelUrl?: string;
worktree?: WorktreeInfo;
eeWorktreePath?: string;
logger?: Logger;
}
// No default config needed since commitHash is always required
export class EphemeralBackend {
private config: Config;
private resources: SpawnedResources = { dbContainerId: "" };
getDbPort(): number {
return this.config.dbPort;
}
getServerPort(): number {
return this.config.serverPort;
}
getLogFilePath(): string | undefined {
return this.resources.logger?.getLogFilePath();
}
constructor(config: Config) {
this.config = config;
}
async spawn(): Promise<{
tunnelUrl: string;
}> {
try {
// Initialize logger for this ephemeral backend
this.resources.logger = new Logger(this.config.commitHash, "backend");
this.resources.logger.log("🚀 Starting ephemeral backend...");
this.resources.logger.log(`📊 Database port: ${this.config.dbPort}`);
this.resources.logger.log(`🌐 Server port: ${this.config.serverPort}`);
this.resources.logger.log(`📌 Commit hash: ${this.config.commitHash}`);
await this.startCloudflared();
if (!this.resources.tunnelUrl)
throw new Error("Cloudflare tunnel URL not available");
await this.acquireWorktree();
await this.setupEECode();
await this.spawnPostgres();
await this.waitForPostgres();
if (!this.config.skipBuild) {
await this.buildBackend();
} else {
this.resources.logger?.log(
"\n⏭️ Skipping backend build (using existing binary)"
);
}
await this.startBackend();
// Release the worktree back to the pool now that the backend is running
// The binary is already compiled and running, so other spawns can reuse this worktree
if (this.resources.worktree) {
this.resources.logger?.log(
"\n♻️ Releasing worktree back to pool (backend is now running)..."
);
await this.config.worktreePool.release(this.resources.worktree.id);
// Keep the reference for cleanup but mark it as released
this.resources.worktree = undefined;
}
this.resources.logger?.log("\n✅ Ephemeral backend is ready!");
this.resources.logger?.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`);
this.resources.logger?.log(
`📄 Log file: ${this.resources.logger.getLogFilePath()}`
);
return {
tunnelUrl: this.resources.tunnelUrl,
};
} catch (error) {
this.resources.logger?.error(
`❌ Error spawning ephemeral backend: ${error}`
);
throw error;
}
}
private async acquireWorktree(): Promise<void> {
this.resources.logger?.log("\n📂 Acquiring worktree from pool...");
// Acquire a worktree from the pool
this.resources.worktree = await this.config.worktreePool.acquire(
this.config.commitHash
);
this.resources.logger?.log(
`✓ Worktree acquired: ${this.resources.worktree.path}`
);
}
private async setupEECode(): Promise<void> {
this.resources.logger?.log("\n🔐 Setting up Enterprise Edition code...");
if (!this.resources.worktree) {
throw new Error("Worktree not acquired");
}
const worktreePath = this.resources.worktree.path;
const eeRefPath = `${worktreePath}/backend/ee-repo-ref.txt`;
const eeWorktreePath = this.config.worktreePool.getEEWorktreePath(
this.resources.worktree
);
this.resources.eeWorktreePath = eeWorktreePath;
// Read the EE commit hash from ee-repo-ref.txt
this.resources.logger?.log(
` Reading EE commit reference from ${eeRefPath}`
);
let eeCommitHash: string;
try {
const { stdout } = await execAsync(`cat ${eeRefPath}`);
eeCommitHash = stdout.trim();
if (!eeCommitHash) {
throw new Error("ee-repo-ref.txt is empty");
}
this.resources.logger?.log(` ✓ EE commit hash: ${eeCommitHash}`);
} catch (error) {
throw new Error(
`Failed to read ee-repo-ref.txt: ${
error instanceof Error ? error.message : String(error)
}`
);
}
// Remove existing EE private folder if it exists (from previous runs)
this.resources.logger?.log(` Cleaning up any existing EE repository...`);
try {
await execAsync(`rm -rf ${eeWorktreePath}`);
} catch (error) {
// Ignore errors if directory doesn't exist
}
// Clone the windmill-ee-private repo at the specific commit
this.resources.logger?.log(
` Cloning windmill-ee-private at commit ${eeCommitHash}`
);
try {
await execAsync(
`git clone git@github.com:windmill-labs/windmill-ee-private.git ${eeWorktreePath}`,
{
env: {
...process.env,
GIT_SSH_COMMAND: `ssh -i ${process.env.GIT_EE_DEPLOY_KEY_FILE} -o StrictHostKeyChecking=accept-new`,
},
}
);
this.resources.logger?.log(` ✓ Repository cloned to ${eeWorktreePath}`);
} catch (error) {
throw new Error(
`Failed to clone windmill-ee-private: ${
error instanceof Error ? error.message : String(error)
}`
);
}
// Checkout the specific commit
this.resources.logger?.log(` Checking out commit ${eeCommitHash}`);
try {
await execAsync(`git checkout ${eeCommitHash}`, {
cwd: eeWorktreePath,
});
this.resources.logger?.log(` ✓ Checked out commit ${eeCommitHash}`);
} catch (error) {
throw new Error(
`Failed to checkout EE commit: ${
error instanceof Error ? error.message : String(error)
}`
);
}
// Run the substitute_ee_code.sh script to copy EE files
this.resources.logger?.log(
` Running substitute_ee_code.sh to copy EE files`
);
try {
await execAsync(`./substitute_ee_code.sh --copy -d ${eeWorktreePath}`, {
cwd: `${worktreePath}/backend`,
});
this.resources.logger?.log(` ✓ EE code substituted successfully`);
} catch (error) {
throw new Error(
`Failed to substitute EE code: ${
error instanceof Error ? error.message : String(error)
}`
);
}
this.resources.logger?.log("✓ Enterprise Edition code setup complete");
}
private async spawnPostgres(): Promise<void> {
this.resources.logger?.log("\n🐘 Spawning PostgreSQL container...");
this.resources.dbProcess = spawn("docker", [
"run",
"--rm",
"-p",
`${this.config.dbPort}:5432`,
"-e",
"POSTGRES_PASSWORD=changeme",
"-e",
"POSTGRES_DB=windmill",
"postgres:16",
]);
// Capture and log postgres stdout
this.resources.dbProcess.stdout.on("data", (data: Buffer) => {
const output = data.toString().trim();
if (output) {
this.resources.logger?.log(`[postgres] ${output}`);
}
});
// Capture and log postgres stderr
this.resources.dbProcess.stderr.on("data", (data: Buffer) => {
const output = data.toString().trim();
if (output) {
this.resources.logger?.error(`[postgres] ${output}`);
}
});
this.resources.dbProcess.on("close", (code: number) => {
this.resources.logger?.log(`PostgreSQL process exited with code ${code}`);
});
}
private async waitForPostgres(): Promise<void> {
this.resources.logger?.log("⏳ Waiting for PostgreSQL to be ready...");
const maxAttempts = 30;
const delayMs = 1000;
// Determine the host to connect to
// On Linux, we need to use the host's IP or localhost
// On macOS/Windows, host.docker.internal works
const isLinux = process.platform === "linux";
const dbHost = isLinux ? "172.17.0.1" : "host.docker.internal";
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
await execAsync(
`docker run --rm postgres:16 pg_isready -h ${dbHost} -p ${this.config.dbPort} -U postgres`
);
this.resources.logger?.log("✓ PostgreSQL is ready");
return;
} catch (error) {
if (attempt === maxAttempts) {
throw new Error("PostgreSQL failed to start in time");
}
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}
}
private async buildBackend(): Promise<void> {
this.resources.logger?.log(
"\n🔨 Building backend (this may take a while)..."
);
if (!this.resources.worktree) {
throw new Error("Worktree not acquired");
}
// Detect OS to use correct deno_core feature
const isMacOS = process.platform === "darwin";
const env = { ...process.env, SQLX_OFFLINE: "true" };
const features = [
"enterprise",
"enterprise_saml",
"stripe",
"embedding",
"parquet",
"prometheus",
"openidconnect",
"cloud",
"jemalloc",
"agent_worker_server",
"tantivy",
"license",
"http_trigger",
"zip",
"oauth2",
"kafka",
"sqs_trigger",
"nats",
"otel",
"dind",
"postgres_trigger",
"mqtt_trigger",
"gcp_trigger",
"websocket",
"smtp",
"all_languages",
"private",
isMacOS ? "deno_core_mac" : "deno_core",
"mcp",
].join(",");
return new Promise((resolve, reject) => {
const backendDir = `${this.resources.worktree?.path}/backend`;
const buildProcess = spawn(
"cargo",
["build", "--features", features, "--release"],
{ cwd: backendDir, env }
);
buildProcess.stdout.on("data", (data) => {
const output = data.toString().trim();
if (output) {
this.resources.logger?.log(`[build] ${output}`);
}
});
buildProcess.stderr.on("data", (data) => {
const output = data.toString().trim();
if (output) {
this.resources.logger?.error(`[build] ${output}`);
}
});
buildProcess.on("close", (code) => {
if (code === 0) {
this.resources.logger?.log("✓ Backend built successfully");
resolve();
} else {
this.resources.logger?.error(`Build failed with code ${code}`);
reject(new Error(`Build failed with code ${code}`));
}
});
});
}
private async startBackend(): Promise<void> {
this.resources.logger?.log("\n🚀 Starting Windmill backend...");
if (!this.resources.worktree) {
throw new Error("Worktree not acquired");
}
const env = {
// DO NOT PASS the entire process.env to avoid leaking sensitive information
...(process.env.LICENSE_KEY && { LICENSE_KEY: process.env.LICENSE_KEY }),
DATABASE_URL: `postgres://postgres:changeme@localhost:${this.config.dbPort}/windmill?sslmode=disable`,
PORT: this.config.serverPort.toString(),
};
const releaseDir = `${this.resources.worktree.path}/backend/target/release`;
// Get sandbox user UID/GID for proper sandboxing
const sandboxUid = await this.getSandboxUid();
const sandboxGid = await this.getSandboxGid();
// Use bwrap for sandboxing the windmill process
this.resources.backendProcess = spawn(
"bwrap",
[
...["--ro-bind", "/", "/"],
...["--bind", "/home/sandbox", "/home/sandbox"],
...["--bind", "/tmp", "/tmp"],
...["--dev", "/dev"],
...["--proc", "/proc"],
"--unshare-user",
...["--uid", sandboxUid],
...["--gid", sandboxGid],
`${releaseDir}/windmill`,
],
{
env,
}
);
// Wait for backend to be ready by watching for "Windmill Enterprise Edition" in output
const backendReady = new Promise<void>((resolve, reject) => {
// Capture and log backend stdout, watching for ready signal
this.resources.backendProcess.stdout.on("data", (data: any) => {
const output = data.toString().trim();
const timeout = setTimeout(() => {
reject(new Error("Backend failed to start in time"));
}, 30000); // 30 seconds timeout
if (output) {
this.resources.logger?.log(`[backend] ${output}`);
if (output.includes("Windmill Enterprise Edition")) {
clearTimeout(timeout);
resolve();
}
}
});
});
// Capture and log backend stderr
this.resources.backendProcess.stderr.on("data", (data: Buffer) => {
const output = data.toString().trim();
if (output) {
this.resources.logger?.error(`[backend] ${output}`);
}
});
this.resources.backendProcess.on("close", (code: number) => {
this.resources.logger?.log(`Backend process exited with code ${code}`);
});
// Wait for backend to be ready
await backendReady;
this.resources.logger?.log("✓ Backend is running");
// Wait 2 additional seconds to be sure
await new Promise((resolve) => setTimeout(resolve, 2000));
// Setup admin password
await this.setupAdminPassword();
}
private async startCloudflared(): Promise<void> {
if (process.env.SKIP_CLOUDFLARED) {
this.config.onCloudflaredUrl?.("SKIP_CLOUDFLARED");
return;
}
this.resources.logger?.log("\n🌐 Starting Cloudflare tunnel...");
return new Promise((resolve, reject) => {
this.resources.cloudflaredProcess = spawn("cloudflared", [
"tunnel",
"--url",
`http://localhost:${this.config.serverPort}`,
"--config",
"/dev/null",
]);
const rl = readline.createInterface({
input: this.resources.cloudflaredProcess.stdout,
});
rl.on("line", (line: string) => {
if (line.trim()) {
this.resources.logger?.log(`[cloudflared] ${line}`);
}
});
this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => {
const line = data.toString();
if (line.trim()) {
this.resources.logger?.error(`[cloudflared] ${line.trim()}`);
}
const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/);
if (match) {
this.resources.tunnelUrl = match[1];
this.resources.logger?.log(
`✓ Tunnel URL: ${this.resources.tunnelUrl}`
);
this.config.onCloudflaredUrl?.(this.resources.tunnelUrl);
resolve();
}
});
this.resources.cloudflaredProcess.on("close", (code: number) => {
this.resources.logger?.log(
`Cloudflared process exited with code ${code}`
);
});
// Timeout if we can't find the URL in 30 seconds
setTimeout(() => {
if (!this.resources.tunnelUrl) {
this.resources.logger?.error(
"Failed to extract Cloudflare tunnel URL (timeout)"
);
reject(new Error("Failed to extract Cloudflare tunnel URL"));
}
}, 30000);
});
}
private async getSandboxUid(): Promise<string> {
try {
const { stdout } = await execAsync("id -u sandbox");
return stdout.trim();
} catch (error) {
this.resources.logger?.error(
`Failed to get sandbox UID: ${
error instanceof Error ? error.message : String(error)
}`
);
throw new Error("Failed to get sandbox user UID");
}
}
private async getSandboxGid(): Promise<string> {
try {
const { stdout } = await execAsync("id -g sandbox");
return stdout.trim();
} catch (error) {
this.resources.logger?.error(
`Failed to get sandbox GID: ${
error instanceof Error ? error.message : String(error)
}`
);
throw new Error("Failed to get sandbox user GID");
}
}
private async setupAdminPassword(): Promise<void> {
this.resources.logger?.log("\n🔑 Setting up admin password...");
const baseUrl = `http://localhost:${this.config.serverPort}`;
try {
// Login with default credentials
this.resources.logger?.log(" Logging in with default credentials...");
const loginResponse = await fetch(`${baseUrl}/api/auth/login`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
email: "admin@windmill.dev",
password: "changeme",
}),
});
if (!loginResponse.ok) {
throw new Error(
`Login failed with status ${
loginResponse.status
}: ${await loginResponse.text()}`
);
}
const token = await loginResponse.text();
this.resources.logger?.log(" ✓ Logged in successfully");
// Set new admin password
this.resources.logger?.log(" Setting new admin password...");
const setPasswordResponse = await fetch(
`${baseUrl}/api/users/setpassword`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({
password: this.config.adminPwd,
}),
}
);
if (!setPasswordResponse.ok) {
throw new Error(
`Set password failed with status ${
setPasswordResponse.status
}: ${await setPasswordResponse.text()}`
);
}
this.resources.logger?.log("✓ Admin password configured successfully");
} catch (error) {
this.resources.logger?.error(
`Failed to setup admin password: ${
error instanceof Error ? error.message : String(error)
}`
);
throw error;
}
}
async cleanup(): Promise<void> {
this.resources.logger?.log("\n🧹 Cleaning up resources...");
// Kill backend process
if (this.resources.backendProcess) {
this.resources.logger?.log(" Stopping backend...");
try {
this.resources.backendProcess.kill("SIGTERM");
// Give it a moment to gracefully shutdown
await new Promise((resolve) => setTimeout(resolve, 1000));
// Force kill if still running
this.resources.backendProcess.kill("SIGKILL");
} catch (error) {
// Process might already be dead
}
}
// Kill cloudflared process
if (this.resources.cloudflaredProcess) {
this.resources.logger?.log(" Stopping cloudflared...");
try {
this.resources.cloudflaredProcess.kill("SIGTERM");
await new Promise((resolve) => setTimeout(resolve, 1000));
this.resources.cloudflaredProcess.kill("SIGKILL");
} catch (error) {
// Process might already be dead
}
}
// Kill PostgreSQL process
if (this.resources.dbProcess) {
this.resources.logger?.log(" Stopping PostgreSQL container...");
try {
this.resources.dbProcess.kill("SIGTERM");
await new Promise((resolve) => setTimeout(resolve, 1000));
this.resources.dbProcess.kill("SIGKILL");
this.resources.logger?.log(" ✓ PostgreSQL container stopped");
} catch (error) {
console.error(" Failed to stop PostgreSQL container:", error);
}
}
// Remove EE private repository clone
if (this.resources.eeWorktreePath) {
this.resources.logger?.log(" Removing EE private repository clone...");
try {
await execAsync(`rm -rf ${this.resources.eeWorktreePath}`);
this.resources.logger?.log(" ✓ EE private repository clone removed");
} catch (error) {
console.error(" Failed to remove EE private repository clone:", error);
}
}
// Release git worktree back to pool (do not delete it)
// Note: worktree might already be released if backend started successfully
if (this.resources.worktree) {
this.resources.logger?.log(" Releasing worktree back to pool...");
try {
await this.config.worktreePool.release(this.resources.worktree.id);
this.resources.logger?.log(" ✓ Worktree released for reuse");
} catch (error) {
console.error(" Failed to release worktree:", error);
}
} else {
this.resources.logger?.log(" ✓ Worktree already released");
}
this.config.onCleanup?.();
this.resources.logger?.log("✅ Cleanup complete");
}
}

15
ephemeral-backends/tsconfig.json Normal file
View File

@@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "node",
"esModuleInterop": true,
"strict": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"outDir": "./dist",
"rootDir": "."
},
"include": ["*.ts"],
"exclude": ["node_modules", "dist"]
}

34
ephemeral-backends/windmill-ephemeral-manager.service Normal file
View File

@@ -0,0 +1,34 @@
[Unit]
Description=Windmill Ephemeral Backend Manager
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=sandbox
Group=sandbox
WorkingDirectory=/home/sandbox/ephemeral-backend/windmill
EnvironmentFile=/etc/windmill-ephemeral-manager/.env
# Start the manager
ExecStart=bun run ephemeral-backends/manager
# Restart policy
Restart=on-failure
RestartSec=10s
# Resource limits
LimitNOFILE=65536
LimitNPROC=4096
# Security settings
NoNewPrivileges=true
PrivateTmp=true
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=windmill-ephemeral-manager
[Install]
WantedBy=multi-user.target

231
ephemeral-backends/worktree-pool.ts Normal file
View File

@@ -0,0 +1,231 @@
import { promisify } from "util";
import { exec } from "child_process";
import path from "path";
const execAsync = promisify(exec);
export interface WorktreeInfo {
id: number;
path: string;
inUse: boolean;
currentCommit?: string;
}
export class WorktreePool {
private worktrees: Map<number, WorktreeInfo> = new Map();
private baseWorktreePath: string;
private nextId: number = 0;
constructor(baseWorktreePath: string = "../windmill-ephemeral-backends") {
this.baseWorktreePath = path.resolve(baseWorktreePath);
}
/**
* Initialize the pool by discovering existing worktrees
*/
async initialize(): Promise<void> {
console.log("🔍 Initializing worktree pool...");
// Ensure base directory exists
try {
await execAsync(`mkdir -p ${this.baseWorktreePath}`);
} catch (error) {
// Directory might already exist
}
// Discover existing worktrees
await this.discoverExistingWorktrees();
console.log(
`✓ Worktree pool initialized with ${this.worktrees.size} existing worktree(s)`
);
}
/**
* Discover existing worktrees from git worktree list
*/
private async discoverExistingWorktrees(): Promise<void> {
try {
const { stdout } = await execAsync("git worktree list --porcelain");
const lines = stdout.split("\n");
let currentWorktreePath: string | null = null;
let isMainWorktree = false;
for (const line of lines) {
if (line.startsWith("worktree ")) {
currentWorktreePath = line.substring("worktree ".length);
isMainWorktree = false;
} else if (line.startsWith("branch ")) {
// Main worktree has a branch entry, so this is NOT a detached HEAD worktree
// We want to skip the main worktree
isMainWorktree = true;
} else if (line === "" && currentWorktreePath) {
// End of worktree entry
// Check if this worktree is in our base path and not the main worktree
if (
currentWorktreePath.startsWith(this.baseWorktreePath) &&
!isMainWorktree
) {
// Extract the ID from the path (e.g., .../worktree-0 -> 0)
const match = currentWorktreePath.match(/worktree-(\d+)$/);
if (match) {
const id = parseInt(match[1], 10);
this.worktrees.set(id, {
id,
path: currentWorktreePath,
inUse: false,
currentCommit: undefined,
});
if (id >= this.nextId) {
this.nextId = id + 1;
}
console.log(` Found existing worktree: ${currentWorktreePath}`);
}
}
currentWorktreePath = null;
isMainWorktree = false;
}
}
} catch (error) {
// If git worktree list fails, we'll start with an empty pool
console.log(" No existing worktrees found");
}
}
/**
* Acquire a worktree for the given commit hash
* Returns a free worktree or creates a new one if needed
*/
async acquire(commitHash: string): Promise<WorktreeInfo> {
console.log(`\n📂 Acquiring worktree for commit ${commitHash}...`);
// Try to find a free worktree
for (const worktree of this.worktrees.values()) {
if (!worktree.inUse) {
console.log(` Using existing worktree: ${worktree.path}`);
await this.prepareWorktree(worktree, commitHash);
worktree.inUse = true;
worktree.currentCommit = commitHash;
return worktree;
}
}
// No free worktrees, create a new one
console.log(" No free worktrees available, creating new one...");
const newWorktree = await this.createWorktree(commitHash);
newWorktree.inUse = true;
newWorktree.currentCommit = commitHash;
this.worktrees.set(newWorktree.id, newWorktree);
return newWorktree;
}
/**
* Prepare a worktree for use by discarding changes and checking out the commit
*/
private async prepareWorktree(
worktree: WorktreeInfo,
commitHash: string
): Promise<void> {
console.log(` Preparing worktree at ${worktree.path}...`);
try {
// Discard all local changes
await execAsync("git reset --hard", { cwd: worktree.path });
await execAsync("git clean -fd", { cwd: worktree.path });
// Fetch to ensure we have the latest commits
await execAsync("git fetch origin", { cwd: worktree.path });
// Checkout the target commit
await execAsync(`git checkout ${commitHash}`, { cwd: worktree.path });
console.log(` ✓ Worktree prepared and checked out to ${commitHash}`);
} catch (error) {
throw new Error(
`Failed to prepare worktree at ${worktree.path}: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
/**
* Create a new worktree for the given commit hash
*/
private async createWorktree(commitHash: string): Promise<WorktreeInfo> {
const id = this.nextId++;
const worktreePath = path.join(this.baseWorktreePath, `worktree-${id}`);
console.log(` Creating new worktree at ${worktreePath}...`);
try {
// First, fetch to ensure we have the commit
await execAsync(`git fetch origin ${commitHash}`);
// Create the worktree at the specific commit
// Note: This command must be run from the main repository directory
await execAsync(`git worktree add ${worktreePath} ${commitHash}`);
console.log(` ✓ Worktree created at ${worktreePath}`);
return {
id,
path: worktreePath,
inUse: false,
currentCommit: undefined,
};
} catch (error) {
throw new Error(
`Failed to create worktree at ${worktreePath}: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
/**
* Release a worktree back to the pool
* The worktree is marked as available but NOT deleted
*/
async release(worktreeId: number): Promise<void> {
const worktree = this.worktrees.get(worktreeId);
if (!worktree) {
console.warn(`⚠️ Worktree ${worktreeId} not found in pool`);
return;
}
console.log(`\n📂 Releasing worktree ${worktreeId} back to pool...`);
// Mark as available
worktree.inUse = false;
worktree.currentCommit = undefined;
console.log(`✓ Worktree ${worktreeId} is now available for reuse`);
}
/**
* Get the EE worktree path for a given worktree
*/
getEEWorktreePath(worktree: WorktreeInfo): string {
return `${worktree.path}_private`;
}
/**
* Get pool statistics
*/
getStats(): {
total: number;
inUse: number;
available: number;
} {
const total = this.worktrees.size;
const inUse = Array.from(this.worktrees.values()).filter(
(w) => w.inUse
).length;
return {
total,
inUse,
available: total - inUse,
};
}
}

6
frontend/src/routes/(root)/(logged)/+layout.svelte
View File

@@ -71,6 +71,12 @@
children?: import('svelte').Snippet
}
const remoteUrlParam = $page.url.searchParams.get('remote_url')
if (remoteUrlParam) {
document.cookie = `remote_url=${remoteUrlParam}; path=/; secure; samesite=strict`
$page.url.searchParams.delete('remote_url')
}
let { children }: Props = $props()
OpenAPI.WITH_CREDENTIALS = true
let menuOpen = $state(false)

20
functions/api/[[path]].ts
View File

@@ -13,7 +13,15 @@ export async function onRequest(context) {
try {
const url = new URL(request.url);
url.hostname = "app.windmill.dev";
// Check for remote_url cookie
const cookies = request.headers.get("cookie") || "";
const remoteUrlMatch = cookies.match(/remote_url=([^;]+)/);
const targetHostname = remoteUrlMatch
? decodeURIComponent(remoteUrlMatch[1])
: "app.windmill.dev";
url.hostname = targetHostname;
const res = await fetch(url.toString(), {
method: request.method,
body: request.body,
@@ -21,14 +29,18 @@ export async function onRequest(context) {
redirect: "manual",
});
const newResponse = new Response(res.body, res);
// Extract domain from target hostname for cookie domain replacement
const targetDomain = targetHostname.split(".").slice(-2).join(".");
newResponse.headers.set(
"set-cookie",
"token" +
newResponse.headers
(newResponse.headers
.get("set-cookie")
?.replace("Domain=windmill.dev;", "")
?.replace(`Domain=${targetDomain};`, "")
?.split("token")
.pop() ?? ""
?.pop() ?? "")
);
newResponse.headers.set("Cross-Origin-Resource-Policy", "cross-origin");