Fullstack CI Preview (#7665)
* update cf worker hostname
* set remote_url cookie from param
* ephemeral backends v1
* nit
* Run queue server
* ntis
* timeout
* better db process management
* commit hash and worktree
* nit use map
* nit
* err handling
* Revert "err handling"
This reverts commit 19de00c0c0.
* nits
* auto cleanup
* Ephemeral backend command action
* remove checkout
* checkout ee repo
* nits
* process.env.GIT_EE_DEPLOY_KEY_FILE
* resumeURLs logic
* nit
* use windmill flow for ephemeral backend action
* fixes
* new token
* worktree pools
* Delete GH secret on cleanup
* linux deploy
* nit
* nit
* unhandled promises
* nit
* fix docker bridge IP on linux
* pass cf_frontend_url to wmill flow
* git fetch
* release worktree when binary started
* send error
* logger
* logging
* logging 2
* delete log files periodically
* redirect to raw app with logs
* CORS
* MANAGER_AUTH_TOKEN
* Check organization membership
* nit
* bwrap
* nit
* return timeoutAt in resumeUrl
* nit
* Change password
* nit remove https
This commit is contained in:
Diego Imbert
131
.github/workflows/spawn-ephemeral-backend.yml
vendored
131
.github/workflows/spawn-ephemeral-backend.yml
vendored
@@ -22,6 +22,49 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
- name: Check organization membership
|
||||||
|
if: github.event_name == 'issue_comment'
|
||||||
|
id: check-org-member
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const commenter = context.payload.comment.user.login;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const membership = await github.rest.orgs.checkMembershipForUser({
|
||||||
|
org: 'windmill-labs',
|
||||||
|
username: commenter
|
||||||
|
});
|
||||||
|
|
||||||
|
core.setOutput('is_member', 'true');
|
||||||
|
console.log(`✓ User ${commenter} is a member of windmill-labs`);
|
||||||
|
} catch (error) {
|
||||||
|
core.setOutput('is_member', 'false');
|
||||||
|
console.log(`✗ User ${commenter} is not a member of windmill-labs`);
|
||||||
|
}
|
||||||
|
|
||||||
|
- name: Post unauthorized comment
|
||||||
|
if: |
|
||||||
|
github.event_name == 'issue_comment' &&
|
||||||
|
steps.check-org-member.outputs.is_member == 'false'
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
await github.rest.issues.createComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: context.issue.number,
|
||||||
|
body: `❌ Unauthorized: Only members of the windmill-labs organization can spawn ephemeral backends.`
|
||||||
|
});
|
||||||
|
|
||||||
|
- name: Fail if not org member
|
||||||
|
if: |
|
||||||
|
github.event_name == 'issue_comment' &&
|
||||||
|
steps.check-org-member.outputs.is_member == 'false'
|
||||||
|
run: |
|
||||||
|
echo "Error: User is not a member of windmill-labs organization"
|
||||||
|
exit 1
|
||||||
|
|
||||||
- name: Get PR details
|
- name: Get PR details
|
||||||
id: pr-details
|
id: pr-details
|
||||||
uses: actions/github-script@v7
|
uses: actions/github-script@v7
|
||||||
@@ -36,40 +79,80 @@ jobs:
|
|||||||
repo: context.repo.repo,
|
repo: context.repo.repo,
|
||||||
pull_number: prNumber
|
pull_number: prNumber
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Get branch name and format it for Cloudflare Pages
|
||||||
|
// Replace '/' with '-' for the URL
|
||||||
|
const branchName = pr.data.head.ref;
|
||||||
|
const formattedBranch = branchName.replace(/\//g, '-');
|
||||||
|
const cfFrontendUrl = `https://${formattedBranch}.windmill.pages.dev`;
|
||||||
|
|
||||||
core.setOutput('commit_hash', pr.data.head.sha);
|
core.setOutput('commit_hash', pr.data.head.sha);
|
||||||
core.setOutput('pr_number', context.issue.number);
|
core.setOutput('pr_number', prNumber);
|
||||||
|
core.setOutput('branch_name', branchName);
|
||||||
|
core.setOutput('cf_frontend_url', cfFrontendUrl);
|
||||||
|
|
||||||
- name: Trigger Windmill flow
|
- name: Check manager URL
|
||||||
id: trigger-flow
|
id: check-manager-url
|
||||||
run: |
|
run: |
|
||||||
RESPONSE=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \
|
if [ -z "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}" ]; then
|
||||||
-H "Authorization: Bearer ${{ secrets.WINDMILL_TOKEN }}" \
|
echo "manager_url_set=false" >> $GITHUB_OUTPUT
|
||||||
-H "Content-Type: application/json" \
|
else
|
||||||
-d '{
|
echo "manager_url_set=true" >> $GITHUB_OUTPUT
|
||||||
"manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}",
|
|
||||||
"commit_hash": "${{ steps.pr-details.outputs.commit_hash }}",
|
|
||||||
"pr_number": ${{ steps.pr-details.outputs.pr_number }}
|
|
||||||
}')
|
|
||||||
|
|
||||||
JOB_UUID=$(echo "$RESPONSE" | jq -r '.id // empty')
|
|
||||||
|
|
||||||
if [ -z "$JOB_UUID" ]; then
|
|
||||||
echo "Failed to get job UUID from response: $RESPONSE"
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT
|
- name: Post error comment if manager not running
|
||||||
|
if: steps.check-manager-url.outputs.manager_url_set == 'false'
|
||||||
- name: Post comment with job link
|
|
||||||
uses: actions/github-script@v7
|
uses: actions/github-script@v7
|
||||||
with:
|
with:
|
||||||
script: |
|
script: |
|
||||||
const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}';
|
const prNumber = context.eventName === 'workflow_dispatch'
|
||||||
const jobUrl = `https://app.windmill.dev/run/${jobUuid}?workspace=windmill-labs`;
|
? Number(context.payload.inputs.pr_number)
|
||||||
|
: context.issue.number;
|
||||||
|
|
||||||
await github.rest.issues.createComment({
|
await github.rest.issues.createComment({
|
||||||
owner: context.repo.owner,
|
owner: context.repo.owner,
|
||||||
repo: context.repo.repo,
|
repo: context.repo.repo,
|
||||||
issue_number: context.issue.number,
|
issue_number: prNumber,
|
||||||
body: `🚀 Ephemeral backend spawning started!\n\nView job progress: ${jobUrl}`
|
body: `❌ Manager URL not set (did you start the ephemeral backend manager?)\n\nThe ephemeral backend manager needs to be running to spawn backends. Please start the manager first.`
|
||||||
|
});
|
||||||
|
|
||||||
|
- name: Fail if manager not running
|
||||||
|
if: steps.check-manager-url.outputs.manager_url_set == 'false'
|
||||||
|
run: |
|
||||||
|
echo "Error: EPHEMERAL_BACKEND_QUEUE_URL secret is not set"
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
- name: Trigger Windmill flow
|
||||||
|
if: steps.check-manager-url.outputs.manager_url_set == 'true'
|
||||||
|
id: trigger-flow
|
||||||
|
run: |
|
||||||
|
JOB_UUID=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \
|
||||||
|
-H "Authorization: Bearer ${{ secrets.WINDMILL_RUN_FLOW_TOKEN }}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}",
|
||||||
|
"commit_hash": "${{ steps.pr-details.outputs.commit_hash }}",
|
||||||
|
"pr_number": ${{ steps.pr-details.outputs.pr_number }},
|
||||||
|
"cf_frontend_url": "${{ steps.pr-details.outputs.cf_frontend_url }}"
|
||||||
|
}' | tr -d '"')
|
||||||
|
|
||||||
|
echo "Job UUID: $JOB_UUID"
|
||||||
|
echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Post comment with job link
|
||||||
|
if: steps.check-manager-url.outputs.manager_url_set == 'true'
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}';
|
||||||
|
const appUrl = `https://app.windmill.dev/public/windmill-labs/a106bad0256c1dfa7a4f9279c42b1a4b#${jobUuid}`;
|
||||||
|
const prNumber = context.eventName === 'workflow_dispatch'
|
||||||
|
? Number(context.payload.inputs.pr_number)
|
||||||
|
: context.issue.number;
|
||||||
|
|
||||||
|
await github.rest.issues.createComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: prNumber,
|
||||||
|
body: `🚀 Spawning new ephemeral backend!\n\n${appUrl}`
|
||||||
});
|
});
|
||||||
|
|||||||
15
ephemeral-backends/.env.template
Normal file
15
ephemeral-backends/.env.template
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Windmill Ephemeral Backend Manager - Environment Variables
|
||||||
|
# Copy this file to /etc/windmill-ephemeral-manager/.env and fill in the values
|
||||||
|
|
||||||
|
# Manager Authentication Token
|
||||||
|
# Required for protecting the manager endpoints from unauthorized access
|
||||||
|
# Generate a secure random token: openssl rand -hex 32
|
||||||
|
MANAGER_AUTH_TOKEN=your-secure-random-token-here
|
||||||
|
|
||||||
|
# GitHub Personal Access Token with 'secrets' scope (Write access)
|
||||||
|
GITHUB_TOKEN=github_pat_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||||
|
|
||||||
|
# Path to the SSH deploy key for accessing windmill-ee-private repository
|
||||||
|
# This should be a read-only deploy key
|
||||||
|
# The file should be owned by the 'sandbox' user with 600 permissions
|
||||||
|
GIT_EE_DEPLOY_KEY_FILE=/home/sandbox/.ssh/windmill_ee_deploy_key
|
||||||
5
ephemeral-backends/.gitignore
vendored
Normal file
5
ephemeral-backends/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
node_modules/
|
||||||
|
dist/
|
||||||
|
*.log
|
||||||
|
.DS_Store
|
||||||
|
.env
|
||||||
35
ephemeral-backends/DEPLOYMENT.md
Normal file
35
ephemeral-backends/DEPLOYMENT.md
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
# Deploying the Ephemeral Backend Manager
|
||||||
|
|
||||||
|
This guide explains how to deploy the Windmill Ephemeral Backend Manager as a systemd service on a Linux machine.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Before running the installation script, ensure you have:
|
||||||
|
|
||||||
|
1. **Linux machine** with systemd (Ubuntu 20.04+, Debian 11+, etc.)
|
||||||
|
2. **User `sandbox`** created with appropriate permissions
|
||||||
|
3. **Repository cloned** to `/home/sandbox/ephemeral-backend/windmill`
|
||||||
|
4. **Required tools installed**:
|
||||||
|
- Git
|
||||||
|
- Docker (with `sandbox` user having access)
|
||||||
|
- Bun (installed for the `sandbox` user)
|
||||||
|
- Rust/Cargo (for building backends)
|
||||||
|
- Bubblewrap (will be installed by the script if missing)
|
||||||
|
- Cloudflared (will be installed by the script if missing)
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
The service runs as user `sandbox` and:
|
||||||
|
|
||||||
|
- Listens on port 8001 for HTTP requests
|
||||||
|
- Creates a Cloudflare tunnel for external access
|
||||||
|
- Updates GitHub Actions secrets with the tunnel URL
|
||||||
|
- Manages a pool of git worktrees for ephemeral backends
|
||||||
|
- Spawns ephemeral backends on-demand
|
||||||
|
- Automatically cleans up resources on shutdown
|
||||||
|
|
||||||
|
The worktree pool enables:
|
||||||
|
|
||||||
|
- Fast incremental compilation (reuses target directories)
|
||||||
|
- Efficient resource usage (no constant creation/deletion)
|
||||||
|
- Automatic discovery of existing worktrees on restart
|
||||||
167
ephemeral-backends/install.sh
Executable file
167
ephemeral-backends/install.sh
Executable file
@@ -0,0 +1,167 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Colors for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
echo "🎛️ Windmill Ephemeral Backend Manager - Installation Script"
|
||||||
|
echo "=========================================================="
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check if running as root
|
||||||
|
if [ "$EUID" -ne 0 ]; then
|
||||||
|
echo -e "${RED}❌ This script must be run as root (use sudo)${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get the actual user who ran sudo (if applicable)
|
||||||
|
ACTUAL_USER="${SUDO_USER:-$(whoami)}"
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
SERVICE_USER="sandbox"
|
||||||
|
SERVICE_NAME="windmill-ephemeral-manager"
|
||||||
|
REPO_DIR="/home/$SERVICE_USER/ephemeral-backend/windmill"
|
||||||
|
ENV_DIR="/etc/$SERVICE_NAME"
|
||||||
|
ENV_FILE="$ENV_DIR/.env"
|
||||||
|
|
||||||
|
echo "📋 Configuration:"
|
||||||
|
echo " Service user: $SERVICE_USER"
|
||||||
|
echo " Repository directory: $REPO_DIR"
|
||||||
|
echo " Environment file: $ENV_FILE"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check if sandbox user exists
|
||||||
|
if ! id "$SERVICE_USER" &>/dev/null; then
|
||||||
|
echo -e "${RED}❌ User '$SERVICE_USER' does not exist${NC}"
|
||||||
|
echo " Create the user first: sudo adduser --system --group $SERVICE_USER"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if repository directory exists
|
||||||
|
if [ ! -d "$REPO_DIR" ]; then
|
||||||
|
echo -e "${RED}❌ Repository directory does not exist: $REPO_DIR${NC}"
|
||||||
|
echo " Clone the repository first as user $SERVICE_USER"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if bun is installed for the service user
|
||||||
|
if ! sudo -u "$SERVICE_USER" bash -c "command -v bun" &>/dev/null; then
|
||||||
|
echo -e "${RED}❌ Bun is not installed for user $SERVICE_USER${NC}"
|
||||||
|
echo " Install bun: curl -fsSL https://bun.sh/install | bash"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if bubblewrap is installed
|
||||||
|
if ! command -v bwrap &>/dev/null; then
|
||||||
|
echo -e "${YELLOW}⚠️ bubblewrap is not installed${NC}"
|
||||||
|
echo " Installing bubblewrap..."
|
||||||
|
apt-get update -qq
|
||||||
|
apt-get install -y -qq bubblewrap
|
||||||
|
echo -e "${GREEN}✓ bubblewrap installed${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if cloudflared is installed
|
||||||
|
if ! command -v cloudflared &>/dev/null; then
|
||||||
|
echo -e "${YELLOW}⚠️ cloudflared is not installed${NC}"
|
||||||
|
echo " Installing cloudflared..."
|
||||||
|
|
||||||
|
# Detect architecture
|
||||||
|
ARCH=$(uname -m)
|
||||||
|
if [ "$ARCH" = "x86_64" ]; then
|
||||||
|
wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb
|
||||||
|
dpkg -i cloudflared-linux-amd64.deb
|
||||||
|
rm cloudflared-linux-amd64.deb
|
||||||
|
elif [ "$ARCH" = "aarch64" ]; then
|
||||||
|
wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-arm64.deb
|
||||||
|
dpkg -i cloudflared-linux-arm64.deb
|
||||||
|
rm cloudflared-linux-arm64.deb
|
||||||
|
else
|
||||||
|
echo -e "${RED}❌ Unsupported architecture: $ARCH${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "${GREEN}✓ cloudflared installed${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if docker is installed and service user has access
|
||||||
|
if ! command -v docker &>/dev/null; then
|
||||||
|
echo -e "${RED}❌ Docker is not installed${NC}"
|
||||||
|
echo " Install docker: https://docs.docker.com/engine/install/"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! sudo -u "$SERVICE_USER" docker ps &>/dev/null; then
|
||||||
|
echo -e "${YELLOW}⚠️ User $SERVICE_USER cannot access Docker${NC}"
|
||||||
|
echo " Adding $SERVICE_USER to docker group..."
|
||||||
|
usermod -aG docker "$SERVICE_USER"
|
||||||
|
echo -e "${GREEN}✓ User added to docker group (may require logout/login)${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create environment directory
|
||||||
|
echo ""
|
||||||
|
echo "📁 Setting up environment..."
|
||||||
|
mkdir -p "$ENV_DIR"
|
||||||
|
chmod 755 "$ENV_DIR"
|
||||||
|
|
||||||
|
# Copy or create environment file
|
||||||
|
if [ ! -f "$ENV_FILE" ]; then
|
||||||
|
if [ -f "$REPO_DIR/ephemeral-backends/.env.template" ]; then
|
||||||
|
cp "$REPO_DIR/ephemeral-backends/.env.template" "$ENV_FILE"
|
||||||
|
echo -e "${YELLOW}⚠️ Environment file created from template: $ENV_FILE${NC}"
|
||||||
|
echo " ${RED}IMPORTANT: Edit this file and fill in the required values!${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}❌ Template file not found: $REPO_DIR/ephemeral-backends/.env.template${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo -e "${GREEN}✓ Environment file already exists: $ENV_FILE${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
chmod 600 "$ENV_FILE"
|
||||||
|
|
||||||
|
# Install systemd service
|
||||||
|
echo ""
|
||||||
|
echo "🔧 Installing systemd service..."
|
||||||
|
cp "$REPO_DIR/ephemeral-backends/windmill-ephemeral-manager.service" "/etc/systemd/system/$SERVICE_NAME.service"
|
||||||
|
chmod 644 "/etc/systemd/system/$SERVICE_NAME.service"
|
||||||
|
echo -e "${GREEN}✓ Service file installed${NC}"
|
||||||
|
|
||||||
|
# Reload systemd
|
||||||
|
echo ""
|
||||||
|
echo "🔄 Reloading systemd..."
|
||||||
|
systemctl daemon-reload
|
||||||
|
echo -e "${GREEN}✓ Systemd reloaded${NC}"
|
||||||
|
|
||||||
|
# Enable service
|
||||||
|
echo ""
|
||||||
|
echo "✅ Enabling service..."
|
||||||
|
systemctl enable "$SERVICE_NAME.service"
|
||||||
|
echo -e "${GREEN}✓ Service enabled (will start on boot)${NC}"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=========================================================="
|
||||||
|
echo -e "${GREEN}✅ Installation complete!${NC}"
|
||||||
|
echo ""
|
||||||
|
echo "📝 Next steps:"
|
||||||
|
echo " 1. Edit the environment file: sudo nano $ENV_FILE"
|
||||||
|
echo " 2. Fill in the required values:"
|
||||||
|
echo " - MANAGER_AUTH_TOKEN (generate with: openssl rand -hex 32)"
|
||||||
|
echo " - GITHUB_TOKEN (GitHub personal access token)"
|
||||||
|
echo " - GIT_EE_DEPLOY_KEY_FILE (path to SSH deploy key)"
|
||||||
|
echo " 3. Ensure the SSH deploy key exists and is readable by $SERVICE_USER"
|
||||||
|
echo " 4. Add MANAGER_AUTH_TOKEN to GitHub repository secrets"
|
||||||
|
echo " 5. Start the service: sudo systemctl start $SERVICE_NAME"
|
||||||
|
echo " 6. Check status: sudo systemctl status $SERVICE_NAME"
|
||||||
|
echo " 7. View logs: sudo journalctl -u $SERVICE_NAME -f"
|
||||||
|
echo ""
|
||||||
|
echo "🔍 Useful commands:"
|
||||||
|
echo " sudo systemctl start $SERVICE_NAME # Start the service"
|
||||||
|
echo " sudo systemctl stop $SERVICE_NAME # Stop the service"
|
||||||
|
echo " sudo systemctl restart $SERVICE_NAME # Restart the service"
|
||||||
|
echo " sudo systemctl status $SERVICE_NAME # Check service status"
|
||||||
|
echo " sudo journalctl -u $SERVICE_NAME -f # Follow logs"
|
||||||
|
echo " sudo journalctl -u $SERVICE_NAME -n 100 # Last 100 log lines"
|
||||||
|
echo ""
|
||||||
131
ephemeral-backends/logger.ts
Normal file
131
ephemeral-backends/logger.ts
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
import {
|
||||||
|
appendFileSync,
|
||||||
|
mkdirSync,
|
||||||
|
unlinkSync,
|
||||||
|
readdirSync,
|
||||||
|
statSync,
|
||||||
|
} from "fs";
|
||||||
|
import path from "path";
|
||||||
|
|
||||||
|
const LOG_DIR = "/tmp/windmill-ephemeral-logs";
|
||||||
|
const LOG_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||||
|
|
||||||
|
export class Logger {
|
||||||
|
private logFilePath: string;
|
||||||
|
private prefix: string;
|
||||||
|
|
||||||
|
constructor(identifier: string, prefix: string = "") {
|
||||||
|
// Sanitize identifier to prevent path traversal
|
||||||
|
const sanitizedId = identifier.replace(/[^a-zA-Z0-9-_]/g, "_");
|
||||||
|
|
||||||
|
// Ensure log directory exists
|
||||||
|
try {
|
||||||
|
mkdirSync(LOG_DIR, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
// Directory might already exist
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logFilePath = path.join(LOG_DIR, `${sanitizedId}.log`);
|
||||||
|
this.prefix = prefix;
|
||||||
|
|
||||||
|
// Write initial log entry
|
||||||
|
const startTime = new Date().toISOString();
|
||||||
|
this.log(`=== Log started at ${startTime} ===`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatMessage(level: string, message: string): string {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const prefixStr = this.prefix ? `[${this.prefix}] ` : "";
|
||||||
|
return `${timestamp} ${level} ${prefixStr}${message}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
log(message: string): void {
|
||||||
|
const formatted = this.formatMessage("stdin ", message);
|
||||||
|
try {
|
||||||
|
appendFileSync(this.logFilePath, formatted);
|
||||||
|
} catch (error) {
|
||||||
|
// Fallback to console if file write fails
|
||||||
|
console.error("Failed to write to log file:", error);
|
||||||
|
console.log(formatted);
|
||||||
|
}
|
||||||
|
// Also log to stdout for systemd journal
|
||||||
|
process.stdout.write(formatted);
|
||||||
|
}
|
||||||
|
|
||||||
|
error(message: string): void {
|
||||||
|
const formatted = this.formatMessage("stderr", message);
|
||||||
|
try {
|
||||||
|
appendFileSync(this.logFilePath, formatted);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to write to log file:", error);
|
||||||
|
console.error(formatted);
|
||||||
|
}
|
||||||
|
// Also log to stderr for systemd journal
|
||||||
|
process.stderr.write(formatted);
|
||||||
|
}
|
||||||
|
|
||||||
|
warn(message: string): void {
|
||||||
|
const formatted = this.formatMessage("warn ", message);
|
||||||
|
try {
|
||||||
|
appendFileSync(this.logFilePath, formatted);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to write to log file:", error);
|
||||||
|
console.warn(formatted);
|
||||||
|
}
|
||||||
|
process.stdout.write(formatted);
|
||||||
|
}
|
||||||
|
|
||||||
|
getLogFilePath(): string {
|
||||||
|
return this.logFilePath;
|
||||||
|
}
|
||||||
|
static getLogFilePathForCommit(commitHash: string): string {
|
||||||
|
// Sanitize commit hash to prevent path traversal
|
||||||
|
const sanitizedHash = commitHash.replace(/[^a-f0-9]/g, "");
|
||||||
|
if (sanitizedHash.length < 7 || sanitizedHash.length > 40) {
|
||||||
|
throw new Error("Invalid commit hash");
|
||||||
|
}
|
||||||
|
return path.join(LOG_DIR, `${sanitizedHash}.log`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean up old log files (older than 24 hours)
|
||||||
|
*/
|
||||||
|
static cleanupOldLogs(): void {
|
||||||
|
try {
|
||||||
|
const files = readdirSync(LOG_DIR);
|
||||||
|
const now = Date.now();
|
||||||
|
let deletedCount = 0;
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
if (!file.endsWith(".log")) continue;
|
||||||
|
|
||||||
|
const filePath = path.join(LOG_DIR, file);
|
||||||
|
try {
|
||||||
|
const stats = statSync(filePath);
|
||||||
|
const ageMs = now - stats.mtimeMs;
|
||||||
|
|
||||||
|
if (ageMs > LOG_RETENTION_MS) {
|
||||||
|
unlinkSync(filePath);
|
||||||
|
deletedCount++;
|
||||||
|
console.log(
|
||||||
|
`Deleted old log file: ${file} (age: ${Math.floor(
|
||||||
|
ageMs / 1000 / 60 / 60
|
||||||
|
)}h)`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Skip files we can't stat or delete
|
||||||
|
console.error(`Failed to process log file ${file}:`, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deletedCount > 0) {
|
||||||
|
console.log(
|
||||||
|
`Cleanup complete: deleted ${deletedCount} old log file(s)`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to cleanup old logs:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
687
ephemeral-backends/manager.ts
Normal file
687
ephemeral-backends/manager.ts
Normal file
@@ -0,0 +1,687 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import { spawn } from "child_process";
|
||||||
|
import * as readline from "readline";
|
||||||
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
|
import { EphemeralBackend } from "./spawn";
|
||||||
|
import { WorktreePool } from "./worktree-pool";
|
||||||
|
import { Logger } from "./logger";
|
||||||
|
import { readFileSync, existsSync } from "fs";
|
||||||
|
|
||||||
|
process.on("unhandledRejection", (err) => {
|
||||||
|
console.error("UNHANDLED PROMISE:", err);
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on("uncaughtException", (err) => {
|
||||||
|
console.error("UNCAUGHT EXCEPTION:", err);
|
||||||
|
});
|
||||||
|
|
||||||
|
const githubToken = process.env.GITHUB_TOKEN;
|
||||||
|
if (!githubToken) {
|
||||||
|
console.log("⚠️ GITHUB_TOKEN environment variable not set");
|
||||||
|
console.log("\n📝 Set a GitHub token with 'secrets' scope:");
|
||||||
|
console.log(" export GITHUB_TOKEN=github_pat_...");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const managerAuthToken = process.env.MANAGER_AUTH_TOKEN;
|
||||||
|
if (!managerAuthToken) {
|
||||||
|
console.log("⚠️ MANAGER_AUTH_TOKEN environment variable not set");
|
||||||
|
console.log("\n📝 Set a secure random token for API authentication:");
|
||||||
|
console.log(" export MANAGER_AUTH_TOKEN=$(openssl rand -hex 32)");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!process.env.GIT_EE_DEPLOY_KEY_FILE) {
|
||||||
|
console.log("⚠️ GIT_EE_DEPLOY_KEY_FILE environment variable not set");
|
||||||
|
console.log("\n📝 Set a read-only SSH deploy key:");
|
||||||
|
console.log(" export GIT_EE_DEPLOY_KEY_FILE=/home/...");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const MANAGER_PORT = 8001;
|
||||||
|
const BACKEND_TIMEOUT_MS = 120 * 60 * 1000; // 2 hours in milliseconds
|
||||||
|
|
||||||
|
interface BackendInfo {
|
||||||
|
backend: EphemeralBackend;
|
||||||
|
timeoutId: NodeJS.Timeout;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ManagerResources {
|
||||||
|
cloudflaredProcess?: any;
|
||||||
|
tunnelUrl?: string;
|
||||||
|
ephemeralBackends: Map<string, BackendInfo>;
|
||||||
|
worktreePool?: WorktreePool;
|
||||||
|
cleanupInterval?: NodeJS.Timeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
class EphemeralBackendManager {
|
||||||
|
private resources: ManagerResources = {
|
||||||
|
ephemeralBackends: new Map(),
|
||||||
|
};
|
||||||
|
private server?: any;
|
||||||
|
|
||||||
|
async start(): Promise<void> {
|
||||||
|
// Setup cleanup handlers early
|
||||||
|
process.on("SIGINT", () => this.cleanup());
|
||||||
|
process.on("SIGTERM", () => this.cleanup());
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log("🎛️ Starting Ephemeral Backend Manager...");
|
||||||
|
console.log(`📊 Manager port: ${MANAGER_PORT}`);
|
||||||
|
|
||||||
|
// Initialize the worktree pool
|
||||||
|
this.resources.worktreePool = new WorktreePool();
|
||||||
|
await this.resources.worktreePool.initialize();
|
||||||
|
|
||||||
|
// Set up periodic log cleanup (every 6 hours)
|
||||||
|
this.resources.cleanupInterval = setInterval(() => {
|
||||||
|
console.log("\n🧹 Running periodic log cleanup...");
|
||||||
|
Logger.cleanupOldLogs();
|
||||||
|
}, 6 * 60 * 60 * 1000);
|
||||||
|
|
||||||
|
await this.startHttpServer();
|
||||||
|
if (!process.env.SKIP_CLOUDFLARED) await this.startCloudflared();
|
||||||
|
if (!process.env.SKIP_SET_GH_SECRET) await this.updateGitHubSecret();
|
||||||
|
|
||||||
|
console.log("\n✅ Manager is ready!");
|
||||||
|
console.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`);
|
||||||
|
|
||||||
|
console.log("\n💡 Press Ctrl+C to stop...");
|
||||||
|
|
||||||
|
// Keep the process running indefinitely
|
||||||
|
await new Promise(() => {}); // Never resolves
|
||||||
|
} catch (error) {
|
||||||
|
console.error("❌ Error starting manager:", error);
|
||||||
|
await this.cleanup();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startHttpServer(): Promise<void> {
|
||||||
|
const self = this;
|
||||||
|
console.log("\n🌐 Starting HTTP server...");
|
||||||
|
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
// Use Bun's built-in HTTP server
|
||||||
|
this.server = Bun.serve({
|
||||||
|
port: MANAGER_PORT,
|
||||||
|
idleTimeout: 30,
|
||||||
|
async fetch(req) {
|
||||||
|
const url = new URL(req.url);
|
||||||
|
|
||||||
|
// CORS headers for app.windmill.dev
|
||||||
|
const origin = req.headers.get("origin");
|
||||||
|
const corsHeaders: Record<string, string> = {};
|
||||||
|
|
||||||
|
if (origin === "https://app.windmill.dev") {
|
||||||
|
corsHeaders["Access-Control-Allow-Origin"] = origin;
|
||||||
|
corsHeaders["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS";
|
||||||
|
corsHeaders["Access-Control-Allow-Headers"] =
|
||||||
|
"Content-Type, Authorization";
|
||||||
|
corsHeaders["Access-Control-Max-Age"] = "86400";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle preflight requests
|
||||||
|
if (req.method === "OPTIONS") {
|
||||||
|
return new Response(null, {
|
||||||
|
status: 204,
|
||||||
|
headers: corsHeaders,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Authentication check function
|
||||||
|
const checkAuth = (): boolean => {
|
||||||
|
const authHeader = req.headers.get("authorization");
|
||||||
|
return authHeader === `Bearer ${managerAuthToken}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Unauthorized response helper
|
||||||
|
const unauthorizedResponse = () => {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
error: "Unauthorized",
|
||||||
|
message: "Valid Bearer token required in Authorization header",
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
status: 401,
|
||||||
|
headers: {
|
||||||
|
...corsHeaders,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"WWW-Authenticate": 'Bearer realm="Manager API"',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Health check endpoint
|
||||||
|
if (url.pathname === "/health") {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
status: "ok",
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
headers: { ...corsHeaders, "Content-Type": "application/json" },
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status endpoint - shows all running backends and worktree pool stats
|
||||||
|
if (url.pathname === "/status") {
|
||||||
|
if (!checkAuth()) {
|
||||||
|
return unauthorizedResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
const backends = Array.from(
|
||||||
|
self.resources.ephemeralBackends.entries()
|
||||||
|
).map(([commitHash, backendInfo]) => {
|
||||||
|
const now = new Date();
|
||||||
|
const timeoutAt = new Date(
|
||||||
|
backendInfo.createdAt.getTime() + BACKEND_TIMEOUT_MS
|
||||||
|
);
|
||||||
|
const timeRemainingMs = timeoutAt.getTime() - now.getTime();
|
||||||
|
const timeRemainingMinutes = Math.floor(
|
||||||
|
timeRemainingMs / 1000 / 60
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
commitHash,
|
||||||
|
shortHash: commitHash.substring(0, 8),
|
||||||
|
createdAt: backendInfo.createdAt.toISOString(),
|
||||||
|
timeoutAt: timeoutAt.toISOString(),
|
||||||
|
timeRemainingMinutes,
|
||||||
|
serverPort: backendInfo.backend.getServerPort(),
|
||||||
|
dbPort: backendInfo.backend.getDbPort(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const worktreePoolStats =
|
||||||
|
self.resources.worktreePool?.getStats() || {
|
||||||
|
total: 0,
|
||||||
|
inUse: 0,
|
||||||
|
available: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
activeBackends: backends.length,
|
||||||
|
backends,
|
||||||
|
worktreePool: worktreePoolStats,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
headers: { ...corsHeaders, "Content-Type": "application/json" },
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match /logs/{commit_hash} - serve log files
|
||||||
|
const logsMatch = url.pathname.match(/^\/logs\/([a-f0-9]+)$/);
|
||||||
|
if (logsMatch && req.method === "GET") {
|
||||||
|
if (!checkAuth()) {
|
||||||
|
return unauthorizedResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
const commitHash = logsMatch[1];
|
||||||
|
|
||||||
|
// Validate commit hash format (7-40 hex characters)
|
||||||
|
if (commitHash.length < 7 || commitHash.length > 40) {
|
||||||
|
return new Response("Invalid commit hash", { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Use the Logger class's secure path resolution
|
||||||
|
const logFilePath = Logger.getLogFilePathForCommit(commitHash);
|
||||||
|
|
||||||
|
// Check if file exists
|
||||||
|
if (!existsSync(logFilePath)) {
|
||||||
|
return new Response("Log file not found", {
|
||||||
|
status: 404,
|
||||||
|
headers: corsHeaders,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the log file
|
||||||
|
const logContent = readFileSync(logFilePath, "utf-8");
|
||||||
|
|
||||||
|
return new Response(logContent, {
|
||||||
|
headers: {
|
||||||
|
...corsHeaders,
|
||||||
|
"Content-Type": "text/plain; charset=utf-8",
|
||||||
|
"X-Commit-Hash": commitHash,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error: any) {
|
||||||
|
console.error(`Error reading log file for ${commitHash}:`, error);
|
||||||
|
return new Response(`Error reading log file: ${error.message}`, {
|
||||||
|
status: 500,
|
||||||
|
headers: corsHeaders,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match /spawn/{commit_hash}
|
||||||
|
const spawnMatch = url.pathname.match(/^\/spawn\/([a-f0-9]+)$/);
|
||||||
|
if (spawnMatch && req.method === "POST") {
|
||||||
|
if (!checkAuth()) {
|
||||||
|
return unauthorizedResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = await req.json();
|
||||||
|
if (typeof body !== "object")
|
||||||
|
return new Response("Invalid JSON body", { status: 400 });
|
||||||
|
let resumeUrl = body?.resume_url;
|
||||||
|
if (typeof resumeUrl !== "string")
|
||||||
|
return new Response("Invalid resume_url", { status: 400 });
|
||||||
|
let cancelUrl = body?.cancel_url;
|
||||||
|
if (typeof cancelUrl !== "string")
|
||||||
|
return new Response("Invalid cancel_url", { status: 400 });
|
||||||
|
const commitHash = spawnMatch[1];
|
||||||
|
console.log(
|
||||||
|
`\n🔹 Received request to spawn ephemeral backend for commit: ${commitHash}`
|
||||||
|
);
|
||||||
|
if (self.resources.ephemeralBackends.has(commitHash)) {
|
||||||
|
throw new Error(`Backend ${commitHash} is already running`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!self.resources.worktreePool) {
|
||||||
|
throw new Error("Worktree pool not initialized");
|
||||||
|
}
|
||||||
|
|
||||||
|
const tunnelUrl = await new Promise<string>((res, err) => {
|
||||||
|
const adminRandomPwd = Math.random()
|
||||||
|
.toString(36)
|
||||||
|
.substring(2, 15);
|
||||||
|
const ephemeralBackend = new EphemeralBackend({
|
||||||
|
dbPort: self.findFreeDbPorts(),
|
||||||
|
serverPort: self.findFreeServerPorts(),
|
||||||
|
skipBuild: !!process.env.SKIP_BACKEND_BUILD,
|
||||||
|
commitHash: commitHash,
|
||||||
|
worktreePool: self.resources.worktreePool!,
|
||||||
|
adminPwd: adminRandomPwd,
|
||||||
|
onCloudflaredUrl: (url) => (res(url), clearTimeout(timeout)),
|
||||||
|
onCleanup: () => {
|
||||||
|
const backendInfo =
|
||||||
|
self.resources.ephemeralBackends.get(commitHash);
|
||||||
|
if (backendInfo) {
|
||||||
|
clearTimeout(backendInfo.timeoutId);
|
||||||
|
self.resources.ephemeralBackends.delete(commitHash);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
function onError(e: any) {
|
||||||
|
ephemeralBackend.cleanup().catch(() => {
|
||||||
|
console.error(
|
||||||
|
`Failed to cleanup backend for commit ${commitHash}`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
clearTimeout(timeout);
|
||||||
|
fetch(resumeUrl, {
|
||||||
|
// Cancel URL doesn't show any relevant info, use resume URL
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
status: "error",
|
||||||
|
commitHash,
|
||||||
|
error: e.message,
|
||||||
|
}),
|
||||||
|
}).catch((e) => {
|
||||||
|
console.error(
|
||||||
|
`Failed to notify cancel URL for commit ${commitHash}:`,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
onError(new Error("Timeout waiting for backend URL"));
|
||||||
|
}, 20000);
|
||||||
|
try {
|
||||||
|
ephemeralBackend
|
||||||
|
.spawn()
|
||||||
|
.then(({ tunnelUrl }) => {
|
||||||
|
if (resumeUrl) {
|
||||||
|
fetch(resumeUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
status: "ready",
|
||||||
|
timeoutAt:
|
||||||
|
(self.resources.ephemeralBackends
|
||||||
|
.get(commitHash)
|
||||||
|
?.createdAt.getTime() ?? 0) + BACKEND_TIMEOUT_MS,
|
||||||
|
commitHash,
|
||||||
|
adminPassword: adminRandomPwd,
|
||||||
|
tunnelUrl,
|
||||||
|
}),
|
||||||
|
}).catch((e) => {
|
||||||
|
onError(e);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((e) => onError(e));
|
||||||
|
|
||||||
|
// Set up 1-hour timeout for automatic cleanup
|
||||||
|
const cleanupTimeoutId = setTimeout(async () => {
|
||||||
|
console.log(
|
||||||
|
`\n⏰ Backend ${commitHash} has reached 1-hour timeout, cleaning up...`
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await ephemeralBackend.cleanup();
|
||||||
|
self.resources.ephemeralBackends.delete(commitHash);
|
||||||
|
console.log(
|
||||||
|
`✓ Backend ${commitHash} cleaned up after timeout`
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
`❌ Failed to cleanup backend ${commitHash} after timeout:`,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, BACKEND_TIMEOUT_MS);
|
||||||
|
|
||||||
|
self.resources.ephemeralBackends.set(commitHash, {
|
||||||
|
backend: ephemeralBackend,
|
||||||
|
timeoutId: cleanupTimeoutId,
|
||||||
|
createdAt: new Date(),
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
onError(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
tunnelUrl,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
headers: { ...corsHeaders, "Content-Type": "application/json" },
|
||||||
|
status: 202,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default 404
|
||||||
|
return new Response("Not Found", {
|
||||||
|
status: 404,
|
||||||
|
headers: corsHeaders,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`✓ HTTP server listening on port ${MANAGER_PORT}`);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startCloudflared(): Promise<void> {
|
||||||
|
console.log("\n🌐 Starting Cloudflare tunnel for manager...");
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
this.resources.cloudflaredProcess = spawn("cloudflared", [
|
||||||
|
"tunnel",
|
||||||
|
"--url",
|
||||||
|
`http://localhost:${MANAGER_PORT}`,
|
||||||
|
"--config",
|
||||||
|
"/dev/null",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: this.resources.cloudflaredProcess.stdout,
|
||||||
|
});
|
||||||
|
|
||||||
|
rl.on("line", (line: string) => {
|
||||||
|
console.log(`[cloudflared] ${line}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => {
|
||||||
|
process.stderr.write(`[cloudflared] ${data}`);
|
||||||
|
const line = data.toString();
|
||||||
|
const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/);
|
||||||
|
if (match) {
|
||||||
|
this.resources.tunnelUrl = match[1];
|
||||||
|
console.log(`✓ Tunnel URL extracted: ${this.resources.tunnelUrl}`);
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.cloudflaredProcess.on("close", (code: number) => {
|
||||||
|
console.log(`Cloudflared process exited with code ${code}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Timeout if we can't find the URL in 30 seconds
|
||||||
|
setTimeout(() => {
|
||||||
|
if (!this.resources.tunnelUrl) {
|
||||||
|
reject(new Error("Failed to extract Cloudflare tunnel URL"));
|
||||||
|
}
|
||||||
|
}, 30000);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async updateGitHubSecret(): Promise<void> {
|
||||||
|
console.log("\n🔐 Updating GitHub Actions secret...");
|
||||||
|
|
||||||
|
if (!this.resources.tunnelUrl) {
|
||||||
|
console.error("❌ No tunnel URL available to update secret");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullUrl = `https://${this.resources.tunnelUrl}`;
|
||||||
|
const repo = "windmill-labs/windmill";
|
||||||
|
const secretName = "EPHEMERAL_BACKEND_QUEUE_URL";
|
||||||
|
|
||||||
|
try {
|
||||||
|
// First, get the repository public key for encrypting the secret
|
||||||
|
console.log(" Fetching repository public key...");
|
||||||
|
const keyResponse = await fetch(
|
||||||
|
`https://api.github.com/repos/${repo}/actions/secrets/public-key`,
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${githubToken}`,
|
||||||
|
Accept: "application/vnd.github+json",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!keyResponse.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to fetch public key: ${keyResponse.statusText}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { key, key_id } = await keyResponse.json();
|
||||||
|
|
||||||
|
// Encrypt the secret using libsodium (via tweetnacl for Bun compatibility)
|
||||||
|
console.log(" Encrypting secret value...");
|
||||||
|
await sodium.ready;
|
||||||
|
|
||||||
|
const messageBytes = new TextEncoder().encode(fullUrl);
|
||||||
|
const keyBytes = sodium.from_base64(key, sodium.base64_variants.ORIGINAL);
|
||||||
|
const encryptedBytes = sodium.crypto_box_seal(messageBytes, keyBytes);
|
||||||
|
const encryptedValue = sodium.to_base64(
|
||||||
|
encryptedBytes,
|
||||||
|
sodium.base64_variants.ORIGINAL
|
||||||
|
);
|
||||||
|
|
||||||
|
// Update the secret
|
||||||
|
console.log(" Updating secret...");
|
||||||
|
const updateResponse = await fetch(
|
||||||
|
`https://api.github.com/repos/${repo}/actions/secrets/${secretName}`,
|
||||||
|
{
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${githubToken}`,
|
||||||
|
Accept: "application/vnd.github+json",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
encrypted_value: encryptedValue,
|
||||||
|
key_id: key_id,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!updateResponse.ok) {
|
||||||
|
const errorText = await updateResponse.text();
|
||||||
|
throw new Error(
|
||||||
|
`Failed to update secret: ${updateResponse.statusText} - ${errorText}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`✓ GitHub secret updated successfully!`);
|
||||||
|
console.log(` Repository: ${repo}`);
|
||||||
|
console.log(` Secret: ${secretName}`);
|
||||||
|
console.log(` Value: ${fullUrl}`);
|
||||||
|
} catch (error: any) {
|
||||||
|
console.error("❌ Failed to update GitHub secret:", error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteGitHubSecret(): Promise<void> {
|
||||||
|
const repo = "windmill-labs/windmill";
|
||||||
|
const secretName = "EPHEMERAL_BACKEND_QUEUE_URL";
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log(" Deleting GitHub Actions secret...");
|
||||||
|
const deleteResponse = await fetch(
|
||||||
|
`https://api.github.com/repos/${repo}/actions/secrets/${secretName}`,
|
||||||
|
{
|
||||||
|
method: "DELETE",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${githubToken}`,
|
||||||
|
Accept: "application/vnd.github+json",
|
||||||
|
"X-GitHub-Api-Version": "2022-11-28",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!deleteResponse.ok) {
|
||||||
|
// 404 is acceptable - secret might not exist
|
||||||
|
if (deleteResponse.status === 404) {
|
||||||
|
console.log(
|
||||||
|
` ✓ Secret ${secretName} does not exist (already deleted)`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const errorText = await deleteResponse.text();
|
||||||
|
throw new Error(
|
||||||
|
`Failed to delete secret: ${deleteResponse.statusText} - ${errorText}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`✓ GitHub secret deleted successfully!`);
|
||||||
|
console.log(` Repository: ${repo}`);
|
||||||
|
console.log(` Secret: ${secretName}`);
|
||||||
|
} catch (error: any) {
|
||||||
|
console.error("❌ Failed to delete GitHub secret:", error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isCleaningUp: boolean = false;
|
||||||
|
private async cleanup(): Promise<void> {
|
||||||
|
if (this.isCleaningUp) return;
|
||||||
|
this.isCleaningUp = true;
|
||||||
|
console.log("\n🧹 Cleaning up manager resources...");
|
||||||
|
|
||||||
|
// Stop periodic cleanup interval
|
||||||
|
if (this.resources.cleanupInterval) {
|
||||||
|
console.log(" Stopping periodic log cleanup...");
|
||||||
|
clearInterval(this.resources.cleanupInterval);
|
||||||
|
this.resources.cleanupInterval = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete GitHub secret
|
||||||
|
if (!process.env.SKIP_SET_GH_SECRET) {
|
||||||
|
console.log(" Deleting GitHub Actions secret...");
|
||||||
|
try {
|
||||||
|
await this.deleteGitHubSecret();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(" Failed to delete GitHub secret:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop HTTP server
|
||||||
|
if (this.server) {
|
||||||
|
console.log(" Stopping HTTP server...");
|
||||||
|
try {
|
||||||
|
this.server.stop();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(" Failed to stop HTTP server:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill cloudflared process
|
||||||
|
if (this.resources.cloudflaredProcess) {
|
||||||
|
console.log(" Stopping cloudflared...");
|
||||||
|
try {
|
||||||
|
this.resources.cloudflaredProcess.kill("SIGTERM");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
this.resources.cloudflaredProcess.kill("SIGKILL");
|
||||||
|
} catch (error) {
|
||||||
|
// Process might already be dead
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [commitHash, backendInfo] of this.resources.ephemeralBackends) {
|
||||||
|
const hash = commitHash.substring(0, 8);
|
||||||
|
console.log(
|
||||||
|
` Cleaning up ephemeral backend ${hash} on port ${backendInfo.backend.getServerPort()}...`
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
clearTimeout(backendInfo.timeoutId); // Clear the timeout before cleanup
|
||||||
|
await backendInfo.backend.cleanup();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
` Failed to clean up backend ${hash} on port ${backendInfo.backend.getServerPort()}:`,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("✅ Cleanup complete");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private findFreeDbPorts(): number {
|
||||||
|
const minPort = 5433;
|
||||||
|
for (let port = minPort; port < minPort + 100; port++) {
|
||||||
|
if (
|
||||||
|
![...this.resources.ephemeralBackends.values()].some(
|
||||||
|
(backendInfo) => port === backendInfo.backend.getDbPort()
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return port;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error("No free DB ports available");
|
||||||
|
}
|
||||||
|
|
||||||
|
private findFreeServerPorts(): number {
|
||||||
|
const minPort = 8002;
|
||||||
|
for (let port = minPort; port < minPort + 100; port++) {
|
||||||
|
if (
|
||||||
|
![...this.resources.ephemeralBackends.values()].some(
|
||||||
|
(backendInfo) => port === backendInfo.backend.getServerPort()
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return port;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error("No free server ports available");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main execution
|
||||||
|
async function main() {
|
||||||
|
const manager = new EphemeralBackendManager();
|
||||||
|
await manager.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error("Fatal error:", error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
88
ephemeral-backends/package-lock.json
generated
Normal file
88
ephemeral-backends/package-lock.json
generated
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
{
|
||||||
|
"name": "windmill-ephemeral-backends",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {
|
||||||
|
"": {
|
||||||
|
"name": "windmill-ephemeral-backends",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/libsodium-wrappers-sumo": "^0.7.8",
|
||||||
|
"libsodium-wrappers-sumo": "^0.8.1"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "^1.3.6",
|
||||||
|
"@types/node": "^20.0.0",
|
||||||
|
"bun-types": "^1.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/bun": {
|
||||||
|
"version": "1.3.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.3.6.tgz",
|
||||||
|
"integrity": "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"bun-types": "1.3.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/libsodium-wrappers": {
|
||||||
|
"version": "0.7.14",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz",
|
||||||
|
"integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/@types/libsodium-wrappers-sumo": {
|
||||||
|
"version": "0.7.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.8.tgz",
|
||||||
|
"integrity": "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/libsodium-wrappers": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/node": {
|
||||||
|
"version": "20.19.30",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.30.tgz",
|
||||||
|
"integrity": "sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"undici-types": "~6.21.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/bun-types": {
|
||||||
|
"version": "1.3.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.3.6.tgz",
|
||||||
|
"integrity": "sha512-OlFwHcnNV99r//9v5IIOgQ9Uk37gZqrNMCcqEaExdkVq3Avwqok1bJFmvGMCkCE0FqzdY8VMOZpfpR3lwI+CsQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/libsodium-sumo": {
|
||||||
|
"version": "0.8.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.8.1.tgz",
|
||||||
|
"integrity": "sha512-q8EhpXKjbzWsLZEY3qOym5nA7emtTe/izt6ziSUbUfE8hk2eDP+7Hy0G0bum65JHDChkdtrImNZTR7N5WuiIXg==",
|
||||||
|
"license": "ISC"
|
||||||
|
},
|
||||||
|
"node_modules/libsodium-wrappers-sumo": {
|
||||||
|
"version": "0.8.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.1.tgz",
|
||||||
|
"integrity": "sha512-xcX1+mQbdwko50AsZWn5r4crZO5H5bHh9ceMHsjBxPyrMn0zFHycms2jcHvXzk71nh4V0rQGCiOjB2ckshd/ew==",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"libsodium-sumo": "^0.8.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/undici-types": {
|
||||||
|
"version": "6.21.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||||
|
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
ephemeral-backends/package.json
Normal file
19
ephemeral-backends/package.json
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "windmill-ephemeral-backends",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Ephemeral backend spawner for Windmill",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun run spawn.ts",
|
||||||
|
"manager": "bun run manager.ts"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@types/libsodium-wrappers-sumo": "^0.7.8",
|
||||||
|
"libsodium-wrappers-sumo": "^0.8.1"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "^1.3.6",
|
||||||
|
"@types/node": "^20.0.0",
|
||||||
|
"bun-types": "^1.0.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
670
ephemeral-backends/spawn.ts
Normal file
670
ephemeral-backends/spawn.ts
Normal file
@@ -0,0 +1,670 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import { spawn, exec } from "child_process";
|
||||||
|
import { promisify } from "util";
|
||||||
|
import * as readline from "readline";
|
||||||
|
import { WorktreePool, WorktreeInfo } from "./worktree-pool";
|
||||||
|
import { Logger } from "./logger";
|
||||||
|
|
||||||
|
const execAsync = promisify(exec);
|
||||||
|
|
||||||
|
export interface Config {
|
||||||
|
dbPort: number;
|
||||||
|
serverPort: number;
|
||||||
|
skipBuild: boolean;
|
||||||
|
commitHash: string;
|
||||||
|
worktreePool: WorktreePool;
|
||||||
|
adminPwd: string;
|
||||||
|
onCloudflaredUrl?: (url: string) => void;
|
||||||
|
onCleanup?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SpawnedResources {
|
||||||
|
dbContainerId: string;
|
||||||
|
dbProcess?: any;
|
||||||
|
backendProcess?: any;
|
||||||
|
cloudflaredProcess?: any;
|
||||||
|
tunnelUrl?: string;
|
||||||
|
worktree?: WorktreeInfo;
|
||||||
|
eeWorktreePath?: string;
|
||||||
|
logger?: Logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
// No default config needed since commitHash is always required
|
||||||
|
|
||||||
|
export class EphemeralBackend {
|
||||||
|
private config: Config;
|
||||||
|
private resources: SpawnedResources = { dbContainerId: "" };
|
||||||
|
|
||||||
|
getDbPort(): number {
|
||||||
|
return this.config.dbPort;
|
||||||
|
}
|
||||||
|
getServerPort(): number {
|
||||||
|
return this.config.serverPort;
|
||||||
|
}
|
||||||
|
getLogFilePath(): string | undefined {
|
||||||
|
return this.resources.logger?.getLogFilePath();
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(config: Config) {
|
||||||
|
this.config = config;
|
||||||
|
}
|
||||||
|
|
||||||
|
async spawn(): Promise<{
|
||||||
|
tunnelUrl: string;
|
||||||
|
}> {
|
||||||
|
try {
|
||||||
|
// Initialize logger for this ephemeral backend
|
||||||
|
this.resources.logger = new Logger(this.config.commitHash, "backend");
|
||||||
|
|
||||||
|
this.resources.logger.log("🚀 Starting ephemeral backend...");
|
||||||
|
this.resources.logger.log(`📊 Database port: ${this.config.dbPort}`);
|
||||||
|
this.resources.logger.log(`🌐 Server port: ${this.config.serverPort}`);
|
||||||
|
this.resources.logger.log(`📌 Commit hash: ${this.config.commitHash}`);
|
||||||
|
|
||||||
|
await this.startCloudflared();
|
||||||
|
if (!this.resources.tunnelUrl)
|
||||||
|
throw new Error("Cloudflare tunnel URL not available");
|
||||||
|
await this.acquireWorktree();
|
||||||
|
await this.setupEECode();
|
||||||
|
await this.spawnPostgres();
|
||||||
|
await this.waitForPostgres();
|
||||||
|
if (!this.config.skipBuild) {
|
||||||
|
await this.buildBackend();
|
||||||
|
} else {
|
||||||
|
this.resources.logger?.log(
|
||||||
|
"\n⏭️ Skipping backend build (using existing binary)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await this.startBackend();
|
||||||
|
|
||||||
|
// Release the worktree back to the pool now that the backend is running
|
||||||
|
// The binary is already compiled and running, so other spawns can reuse this worktree
|
||||||
|
if (this.resources.worktree) {
|
||||||
|
this.resources.logger?.log(
|
||||||
|
"\n♻️ Releasing worktree back to pool (backend is now running)..."
|
||||||
|
);
|
||||||
|
await this.config.worktreePool.release(this.resources.worktree.id);
|
||||||
|
// Keep the reference for cleanup but mark it as released
|
||||||
|
this.resources.worktree = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.resources.logger?.log("\n✅ Ephemeral backend is ready!");
|
||||||
|
this.resources.logger?.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`);
|
||||||
|
this.resources.logger?.log(
|
||||||
|
`📄 Log file: ${this.resources.logger.getLogFilePath()}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
tunnelUrl: this.resources.tunnelUrl,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
this.resources.logger?.error(
|
||||||
|
`❌ Error spawning ephemeral backend: ${error}`
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async acquireWorktree(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n📂 Acquiring worktree from pool...");
|
||||||
|
|
||||||
|
// Acquire a worktree from the pool
|
||||||
|
this.resources.worktree = await this.config.worktreePool.acquire(
|
||||||
|
this.config.commitHash
|
||||||
|
);
|
||||||
|
|
||||||
|
this.resources.logger?.log(
|
||||||
|
`✓ Worktree acquired: ${this.resources.worktree.path}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async setupEECode(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n🔐 Setting up Enterprise Edition code...");
|
||||||
|
|
||||||
|
if (!this.resources.worktree) {
|
||||||
|
throw new Error("Worktree not acquired");
|
||||||
|
}
|
||||||
|
|
||||||
|
const worktreePath = this.resources.worktree.path;
|
||||||
|
const eeRefPath = `${worktreePath}/backend/ee-repo-ref.txt`;
|
||||||
|
const eeWorktreePath = this.config.worktreePool.getEEWorktreePath(
|
||||||
|
this.resources.worktree
|
||||||
|
);
|
||||||
|
this.resources.eeWorktreePath = eeWorktreePath;
|
||||||
|
|
||||||
|
// Read the EE commit hash from ee-repo-ref.txt
|
||||||
|
this.resources.logger?.log(
|
||||||
|
` Reading EE commit reference from ${eeRefPath}`
|
||||||
|
);
|
||||||
|
let eeCommitHash: string;
|
||||||
|
try {
|
||||||
|
const { stdout } = await execAsync(`cat ${eeRefPath}`);
|
||||||
|
eeCommitHash = stdout.trim();
|
||||||
|
if (!eeCommitHash) {
|
||||||
|
throw new Error("ee-repo-ref.txt is empty");
|
||||||
|
}
|
||||||
|
this.resources.logger?.log(` ✓ EE commit hash: ${eeCommitHash}`);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to read ee-repo-ref.txt: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove existing EE private folder if it exists (from previous runs)
|
||||||
|
this.resources.logger?.log(` Cleaning up any existing EE repository...`);
|
||||||
|
try {
|
||||||
|
await execAsync(`rm -rf ${eeWorktreePath}`);
|
||||||
|
} catch (error) {
|
||||||
|
// Ignore errors if directory doesn't exist
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clone the windmill-ee-private repo at the specific commit
|
||||||
|
this.resources.logger?.log(
|
||||||
|
` Cloning windmill-ee-private at commit ${eeCommitHash}`
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await execAsync(
|
||||||
|
`git clone git@github.com:windmill-labs/windmill-ee-private.git ${eeWorktreePath}`,
|
||||||
|
{
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
GIT_SSH_COMMAND: `ssh -i ${process.env.GIT_EE_DEPLOY_KEY_FILE} -o StrictHostKeyChecking=accept-new`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
this.resources.logger?.log(` ✓ Repository cloned to ${eeWorktreePath}`);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to clone windmill-ee-private: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checkout the specific commit
|
||||||
|
this.resources.logger?.log(` Checking out commit ${eeCommitHash}`);
|
||||||
|
try {
|
||||||
|
await execAsync(`git checkout ${eeCommitHash}`, {
|
||||||
|
cwd: eeWorktreePath,
|
||||||
|
});
|
||||||
|
this.resources.logger?.log(` ✓ Checked out commit ${eeCommitHash}`);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to checkout EE commit: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the substitute_ee_code.sh script to copy EE files
|
||||||
|
this.resources.logger?.log(
|
||||||
|
` Running substitute_ee_code.sh to copy EE files`
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await execAsync(`./substitute_ee_code.sh --copy -d ${eeWorktreePath}`, {
|
||||||
|
cwd: `${worktreePath}/backend`,
|
||||||
|
});
|
||||||
|
this.resources.logger?.log(` ✓ EE code substituted successfully`);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to substitute EE code: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.resources.logger?.log("✓ Enterprise Edition code setup complete");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async spawnPostgres(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n🐘 Spawning PostgreSQL container...");
|
||||||
|
|
||||||
|
this.resources.dbProcess = spawn("docker", [
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-p",
|
||||||
|
`${this.config.dbPort}:5432`,
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_PASSWORD=changeme",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=windmill",
|
||||||
|
"postgres:16",
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Capture and log postgres stdout
|
||||||
|
this.resources.dbProcess.stdout.on("data", (data: Buffer) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.log(`[postgres] ${output}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Capture and log postgres stderr
|
||||||
|
this.resources.dbProcess.stderr.on("data", (data: Buffer) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.error(`[postgres] ${output}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.dbProcess.on("close", (code: number) => {
|
||||||
|
this.resources.logger?.log(`PostgreSQL process exited with code ${code}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async waitForPostgres(): Promise<void> {
|
||||||
|
this.resources.logger?.log("⏳ Waiting for PostgreSQL to be ready...");
|
||||||
|
|
||||||
|
const maxAttempts = 30;
|
||||||
|
const delayMs = 1000;
|
||||||
|
|
||||||
|
// Determine the host to connect to
|
||||||
|
// On Linux, we need to use the host's IP or localhost
|
||||||
|
// On macOS/Windows, host.docker.internal works
|
||||||
|
const isLinux = process.platform === "linux";
|
||||||
|
const dbHost = isLinux ? "172.17.0.1" : "host.docker.internal";
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
|
try {
|
||||||
|
await execAsync(
|
||||||
|
`docker run --rm postgres:16 pg_isready -h ${dbHost} -p ${this.config.dbPort} -U postgres`
|
||||||
|
);
|
||||||
|
this.resources.logger?.log("✓ PostgreSQL is ready");
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
if (attempt === maxAttempts) {
|
||||||
|
throw new Error("PostgreSQL failed to start in time");
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildBackend(): Promise<void> {
|
||||||
|
this.resources.logger?.log(
|
||||||
|
"\n🔨 Building backend (this may take a while)..."
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!this.resources.worktree) {
|
||||||
|
throw new Error("Worktree not acquired");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect OS to use correct deno_core feature
|
||||||
|
const isMacOS = process.platform === "darwin";
|
||||||
|
|
||||||
|
const env = { ...process.env, SQLX_OFFLINE: "true" };
|
||||||
|
|
||||||
|
const features = [
|
||||||
|
"enterprise",
|
||||||
|
"enterprise_saml",
|
||||||
|
"stripe",
|
||||||
|
"embedding",
|
||||||
|
"parquet",
|
||||||
|
"prometheus",
|
||||||
|
"openidconnect",
|
||||||
|
"cloud",
|
||||||
|
"jemalloc",
|
||||||
|
"agent_worker_server",
|
||||||
|
"tantivy",
|
||||||
|
"license",
|
||||||
|
"http_trigger",
|
||||||
|
"zip",
|
||||||
|
"oauth2",
|
||||||
|
"kafka",
|
||||||
|
"sqs_trigger",
|
||||||
|
"nats",
|
||||||
|
"otel",
|
||||||
|
"dind",
|
||||||
|
"postgres_trigger",
|
||||||
|
"mqtt_trigger",
|
||||||
|
"gcp_trigger",
|
||||||
|
"websocket",
|
||||||
|
"smtp",
|
||||||
|
"all_languages",
|
||||||
|
"private",
|
||||||
|
isMacOS ? "deno_core_mac" : "deno_core",
|
||||||
|
"mcp",
|
||||||
|
].join(",");
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const backendDir = `${this.resources.worktree?.path}/backend`;
|
||||||
|
const buildProcess = spawn(
|
||||||
|
"cargo",
|
||||||
|
["build", "--features", features, "--release"],
|
||||||
|
{ cwd: backendDir, env }
|
||||||
|
);
|
||||||
|
|
||||||
|
buildProcess.stdout.on("data", (data) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.log(`[build] ${output}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
buildProcess.stderr.on("data", (data) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.error(`[build] ${output}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
buildProcess.on("close", (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
this.resources.logger?.log("✓ Backend built successfully");
|
||||||
|
resolve();
|
||||||
|
} else {
|
||||||
|
this.resources.logger?.error(`Build failed with code ${code}`);
|
||||||
|
reject(new Error(`Build failed with code ${code}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startBackend(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n🚀 Starting Windmill backend...");
|
||||||
|
|
||||||
|
if (!this.resources.worktree) {
|
||||||
|
throw new Error("Worktree not acquired");
|
||||||
|
}
|
||||||
|
|
||||||
|
const env = {
|
||||||
|
// DO NOT PASS the entire process.env to avoid leaking sensitive information
|
||||||
|
...(process.env.LICENSE_KEY && { LICENSE_KEY: process.env.LICENSE_KEY }),
|
||||||
|
DATABASE_URL: `postgres://postgres:changeme@localhost:${this.config.dbPort}/windmill?sslmode=disable`,
|
||||||
|
PORT: this.config.serverPort.toString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const releaseDir = `${this.resources.worktree.path}/backend/target/release`;
|
||||||
|
|
||||||
|
// Get sandbox user UID/GID for proper sandboxing
|
||||||
|
const sandboxUid = await this.getSandboxUid();
|
||||||
|
const sandboxGid = await this.getSandboxGid();
|
||||||
|
|
||||||
|
// Use bwrap for sandboxing the windmill process
|
||||||
|
this.resources.backendProcess = spawn(
|
||||||
|
"bwrap",
|
||||||
|
[
|
||||||
|
...["--ro-bind", "/", "/"],
|
||||||
|
...["--bind", "/home/sandbox", "/home/sandbox"],
|
||||||
|
...["--bind", "/tmp", "/tmp"],
|
||||||
|
...["--dev", "/dev"],
|
||||||
|
...["--proc", "/proc"],
|
||||||
|
"--unshare-user",
|
||||||
|
...["--uid", sandboxUid],
|
||||||
|
...["--gid", sandboxGid],
|
||||||
|
`${releaseDir}/windmill`,
|
||||||
|
],
|
||||||
|
{
|
||||||
|
env,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait for backend to be ready by watching for "Windmill Enterprise Edition" in output
|
||||||
|
const backendReady = new Promise<void>((resolve, reject) => {
|
||||||
|
// Capture and log backend stdout, watching for ready signal
|
||||||
|
this.resources.backendProcess.stdout.on("data", (data: any) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
reject(new Error("Backend failed to start in time"));
|
||||||
|
}, 30000); // 30 seconds timeout
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.log(`[backend] ${output}`);
|
||||||
|
if (output.includes("Windmill Enterprise Edition")) {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Capture and log backend stderr
|
||||||
|
this.resources.backendProcess.stderr.on("data", (data: Buffer) => {
|
||||||
|
const output = data.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
this.resources.logger?.error(`[backend] ${output}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.backendProcess.on("close", (code: number) => {
|
||||||
|
this.resources.logger?.log(`Backend process exited with code ${code}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wait for backend to be ready
|
||||||
|
await backendReady;
|
||||||
|
this.resources.logger?.log("✓ Backend is running");
|
||||||
|
|
||||||
|
// Wait 2 additional seconds to be sure
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||||
|
|
||||||
|
// Setup admin password
|
||||||
|
await this.setupAdminPassword();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startCloudflared(): Promise<void> {
|
||||||
|
if (process.env.SKIP_CLOUDFLARED) {
|
||||||
|
this.config.onCloudflaredUrl?.("SKIP_CLOUDFLARED");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.resources.logger?.log("\n🌐 Starting Cloudflare tunnel...");
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
this.resources.cloudflaredProcess = spawn("cloudflared", [
|
||||||
|
"tunnel",
|
||||||
|
"--url",
|
||||||
|
`http://localhost:${this.config.serverPort}`,
|
||||||
|
"--config",
|
||||||
|
"/dev/null",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: this.resources.cloudflaredProcess.stdout,
|
||||||
|
});
|
||||||
|
|
||||||
|
rl.on("line", (line: string) => {
|
||||||
|
if (line.trim()) {
|
||||||
|
this.resources.logger?.log(`[cloudflared] ${line}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => {
|
||||||
|
const line = data.toString();
|
||||||
|
if (line.trim()) {
|
||||||
|
this.resources.logger?.error(`[cloudflared] ${line.trim()}`);
|
||||||
|
}
|
||||||
|
const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/);
|
||||||
|
if (match) {
|
||||||
|
this.resources.tunnelUrl = match[1];
|
||||||
|
this.resources.logger?.log(
|
||||||
|
`✓ Tunnel URL: ${this.resources.tunnelUrl}`
|
||||||
|
);
|
||||||
|
this.config.onCloudflaredUrl?.(this.resources.tunnelUrl);
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.resources.cloudflaredProcess.on("close", (code: number) => {
|
||||||
|
this.resources.logger?.log(
|
||||||
|
`Cloudflared process exited with code ${code}`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Timeout if we can't find the URL in 30 seconds
|
||||||
|
setTimeout(() => {
|
||||||
|
if (!this.resources.tunnelUrl) {
|
||||||
|
this.resources.logger?.error(
|
||||||
|
"Failed to extract Cloudflare tunnel URL (timeout)"
|
||||||
|
);
|
||||||
|
reject(new Error("Failed to extract Cloudflare tunnel URL"));
|
||||||
|
}
|
||||||
|
}, 30000);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getSandboxUid(): Promise<string> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execAsync("id -u sandbox");
|
||||||
|
return stdout.trim();
|
||||||
|
} catch (error) {
|
||||||
|
this.resources.logger?.error(
|
||||||
|
`Failed to get sandbox UID: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
throw new Error("Failed to get sandbox user UID");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getSandboxGid(): Promise<string> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execAsync("id -g sandbox");
|
||||||
|
return stdout.trim();
|
||||||
|
} catch (error) {
|
||||||
|
this.resources.logger?.error(
|
||||||
|
`Failed to get sandbox GID: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
throw new Error("Failed to get sandbox user GID");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async setupAdminPassword(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n🔑 Setting up admin password...");
|
||||||
|
|
||||||
|
const baseUrl = `http://localhost:${this.config.serverPort}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Login with default credentials
|
||||||
|
this.resources.logger?.log(" Logging in with default credentials...");
|
||||||
|
const loginResponse = await fetch(`${baseUrl}/api/auth/login`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
email: "admin@windmill.dev",
|
||||||
|
password: "changeme",
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!loginResponse.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`Login failed with status ${
|
||||||
|
loginResponse.status
|
||||||
|
}: ${await loginResponse.text()}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = await loginResponse.text();
|
||||||
|
this.resources.logger?.log(" ✓ Logged in successfully");
|
||||||
|
|
||||||
|
// Set new admin password
|
||||||
|
this.resources.logger?.log(" Setting new admin password...");
|
||||||
|
const setPasswordResponse = await fetch(
|
||||||
|
`${baseUrl}/api/users/setpassword`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
password: this.config.adminPwd,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!setPasswordResponse.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`Set password failed with status ${
|
||||||
|
setPasswordResponse.status
|
||||||
|
}: ${await setPasswordResponse.text()}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.resources.logger?.log("✓ Admin password configured successfully");
|
||||||
|
} catch (error) {
|
||||||
|
this.resources.logger?.error(
|
||||||
|
`Failed to setup admin password: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async cleanup(): Promise<void> {
|
||||||
|
this.resources.logger?.log("\n🧹 Cleaning up resources...");
|
||||||
|
|
||||||
|
// Kill backend process
|
||||||
|
if (this.resources.backendProcess) {
|
||||||
|
this.resources.logger?.log(" Stopping backend...");
|
||||||
|
try {
|
||||||
|
this.resources.backendProcess.kill("SIGTERM");
|
||||||
|
// Give it a moment to gracefully shutdown
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
// Force kill if still running
|
||||||
|
this.resources.backendProcess.kill("SIGKILL");
|
||||||
|
} catch (error) {
|
||||||
|
// Process might already be dead
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill cloudflared process
|
||||||
|
if (this.resources.cloudflaredProcess) {
|
||||||
|
this.resources.logger?.log(" Stopping cloudflared...");
|
||||||
|
try {
|
||||||
|
this.resources.cloudflaredProcess.kill("SIGTERM");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
this.resources.cloudflaredProcess.kill("SIGKILL");
|
||||||
|
} catch (error) {
|
||||||
|
// Process might already be dead
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill PostgreSQL process
|
||||||
|
if (this.resources.dbProcess) {
|
||||||
|
this.resources.logger?.log(" Stopping PostgreSQL container...");
|
||||||
|
try {
|
||||||
|
this.resources.dbProcess.kill("SIGTERM");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
this.resources.dbProcess.kill("SIGKILL");
|
||||||
|
this.resources.logger?.log(" ✓ PostgreSQL container stopped");
|
||||||
|
} catch (error) {
|
||||||
|
console.error(" Failed to stop PostgreSQL container:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove EE private repository clone
|
||||||
|
if (this.resources.eeWorktreePath) {
|
||||||
|
this.resources.logger?.log(" Removing EE private repository clone...");
|
||||||
|
try {
|
||||||
|
await execAsync(`rm -rf ${this.resources.eeWorktreePath}`);
|
||||||
|
this.resources.logger?.log(" ✓ EE private repository clone removed");
|
||||||
|
} catch (error) {
|
||||||
|
console.error(" Failed to remove EE private repository clone:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Release git worktree back to pool (do not delete it)
|
||||||
|
// Note: worktree might already be released if backend started successfully
|
||||||
|
if (this.resources.worktree) {
|
||||||
|
this.resources.logger?.log(" Releasing worktree back to pool...");
|
||||||
|
try {
|
||||||
|
await this.config.worktreePool.release(this.resources.worktree.id);
|
||||||
|
this.resources.logger?.log(" ✓ Worktree released for reuse");
|
||||||
|
} catch (error) {
|
||||||
|
console.error(" Failed to release worktree:", error);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.resources.logger?.log(" ✓ Worktree already released");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.config.onCleanup?.();
|
||||||
|
|
||||||
|
this.resources.logger?.log("✅ Cleanup complete");
|
||||||
|
}
|
||||||
|
}
|
||||||
15
ephemeral-backends/tsconfig.json
Normal file
15
ephemeral-backends/tsconfig.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "."
|
||||||
|
},
|
||||||
|
"include": ["*.ts"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
34
ephemeral-backends/windmill-ephemeral-manager.service
Normal file
34
ephemeral-backends/windmill-ephemeral-manager.service
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Windmill Ephemeral Backend Manager
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=sandbox
|
||||||
|
Group=sandbox
|
||||||
|
WorkingDirectory=/home/sandbox/ephemeral-backend/windmill
|
||||||
|
EnvironmentFile=/etc/windmill-ephemeral-manager/.env
|
||||||
|
|
||||||
|
# Start the manager
|
||||||
|
ExecStart=bun run ephemeral-backends/manager
|
||||||
|
|
||||||
|
# Restart policy
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=10s
|
||||||
|
|
||||||
|
# Resource limits
|
||||||
|
LimitNOFILE=65536
|
||||||
|
LimitNPROC=4096
|
||||||
|
|
||||||
|
# Security settings
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=windmill-ephemeral-manager
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
231
ephemeral-backends/worktree-pool.ts
Normal file
231
ephemeral-backends/worktree-pool.ts
Normal file
@@ -0,0 +1,231 @@
|
|||||||
|
import { promisify } from "util";
|
||||||
|
import { exec } from "child_process";
|
||||||
|
import path from "path";
|
||||||
|
|
||||||
|
const execAsync = promisify(exec);
|
||||||
|
|
||||||
|
export interface WorktreeInfo {
|
||||||
|
id: number;
|
||||||
|
path: string;
|
||||||
|
inUse: boolean;
|
||||||
|
currentCommit?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class WorktreePool {
|
||||||
|
private worktrees: Map<number, WorktreeInfo> = new Map();
|
||||||
|
private baseWorktreePath: string;
|
||||||
|
private nextId: number = 0;
|
||||||
|
|
||||||
|
constructor(baseWorktreePath: string = "../windmill-ephemeral-backends") {
|
||||||
|
this.baseWorktreePath = path.resolve(baseWorktreePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the pool by discovering existing worktrees
|
||||||
|
*/
|
||||||
|
async initialize(): Promise<void> {
|
||||||
|
console.log("🔍 Initializing worktree pool...");
|
||||||
|
|
||||||
|
// Ensure base directory exists
|
||||||
|
try {
|
||||||
|
await execAsync(`mkdir -p ${this.baseWorktreePath}`);
|
||||||
|
} catch (error) {
|
||||||
|
// Directory might already exist
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discover existing worktrees
|
||||||
|
await this.discoverExistingWorktrees();
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`✓ Worktree pool initialized with ${this.worktrees.size} existing worktree(s)`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Discover existing worktrees from git worktree list
|
||||||
|
*/
|
||||||
|
private async discoverExistingWorktrees(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execAsync("git worktree list --porcelain");
|
||||||
|
const lines = stdout.split("\n");
|
||||||
|
|
||||||
|
let currentWorktreePath: string | null = null;
|
||||||
|
let isMainWorktree = false;
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith("worktree ")) {
|
||||||
|
currentWorktreePath = line.substring("worktree ".length);
|
||||||
|
isMainWorktree = false;
|
||||||
|
} else if (line.startsWith("branch ")) {
|
||||||
|
// Main worktree has a branch entry, so this is NOT a detached HEAD worktree
|
||||||
|
// We want to skip the main worktree
|
||||||
|
isMainWorktree = true;
|
||||||
|
} else if (line === "" && currentWorktreePath) {
|
||||||
|
// End of worktree entry
|
||||||
|
// Check if this worktree is in our base path and not the main worktree
|
||||||
|
if (
|
||||||
|
currentWorktreePath.startsWith(this.baseWorktreePath) &&
|
||||||
|
!isMainWorktree
|
||||||
|
) {
|
||||||
|
// Extract the ID from the path (e.g., .../worktree-0 -> 0)
|
||||||
|
const match = currentWorktreePath.match(/worktree-(\d+)$/);
|
||||||
|
if (match) {
|
||||||
|
const id = parseInt(match[1], 10);
|
||||||
|
this.worktrees.set(id, {
|
||||||
|
id,
|
||||||
|
path: currentWorktreePath,
|
||||||
|
inUse: false,
|
||||||
|
currentCommit: undefined,
|
||||||
|
});
|
||||||
|
if (id >= this.nextId) {
|
||||||
|
this.nextId = id + 1;
|
||||||
|
}
|
||||||
|
console.log(` Found existing worktree: ${currentWorktreePath}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
currentWorktreePath = null;
|
||||||
|
isMainWorktree = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// If git worktree list fails, we'll start with an empty pool
|
||||||
|
console.log(" No existing worktrees found");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Acquire a worktree for the given commit hash
|
||||||
|
* Returns a free worktree or creates a new one if needed
|
||||||
|
*/
|
||||||
|
async acquire(commitHash: string): Promise<WorktreeInfo> {
|
||||||
|
console.log(`\n📂 Acquiring worktree for commit ${commitHash}...`);
|
||||||
|
|
||||||
|
// Try to find a free worktree
|
||||||
|
for (const worktree of this.worktrees.values()) {
|
||||||
|
if (!worktree.inUse) {
|
||||||
|
console.log(` Using existing worktree: ${worktree.path}`);
|
||||||
|
await this.prepareWorktree(worktree, commitHash);
|
||||||
|
worktree.inUse = true;
|
||||||
|
worktree.currentCommit = commitHash;
|
||||||
|
return worktree;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No free worktrees, create a new one
|
||||||
|
console.log(" No free worktrees available, creating new one...");
|
||||||
|
const newWorktree = await this.createWorktree(commitHash);
|
||||||
|
newWorktree.inUse = true;
|
||||||
|
newWorktree.currentCommit = commitHash;
|
||||||
|
this.worktrees.set(newWorktree.id, newWorktree);
|
||||||
|
return newWorktree;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prepare a worktree for use by discarding changes and checking out the commit
|
||||||
|
*/
|
||||||
|
private async prepareWorktree(
|
||||||
|
worktree: WorktreeInfo,
|
||||||
|
commitHash: string
|
||||||
|
): Promise<void> {
|
||||||
|
console.log(` Preparing worktree at ${worktree.path}...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Discard all local changes
|
||||||
|
await execAsync("git reset --hard", { cwd: worktree.path });
|
||||||
|
await execAsync("git clean -fd", { cwd: worktree.path });
|
||||||
|
|
||||||
|
// Fetch to ensure we have the latest commits
|
||||||
|
await execAsync("git fetch origin", { cwd: worktree.path });
|
||||||
|
|
||||||
|
// Checkout the target commit
|
||||||
|
await execAsync(`git checkout ${commitHash}`, { cwd: worktree.path });
|
||||||
|
|
||||||
|
console.log(` ✓ Worktree prepared and checked out to ${commitHash}`);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to prepare worktree at ${worktree.path}: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new worktree for the given commit hash
|
||||||
|
*/
|
||||||
|
private async createWorktree(commitHash: string): Promise<WorktreeInfo> {
|
||||||
|
const id = this.nextId++;
|
||||||
|
const worktreePath = path.join(this.baseWorktreePath, `worktree-${id}`);
|
||||||
|
|
||||||
|
console.log(` Creating new worktree at ${worktreePath}...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// First, fetch to ensure we have the commit
|
||||||
|
await execAsync(`git fetch origin ${commitHash}`);
|
||||||
|
|
||||||
|
// Create the worktree at the specific commit
|
||||||
|
// Note: This command must be run from the main repository directory
|
||||||
|
await execAsync(`git worktree add ${worktreePath} ${commitHash}`);
|
||||||
|
console.log(` ✓ Worktree created at ${worktreePath}`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
path: worktreePath,
|
||||||
|
inUse: false,
|
||||||
|
currentCommit: undefined,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to create worktree at ${worktreePath}: ${
|
||||||
|
error instanceof Error ? error.message : String(error)
|
||||||
|
}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Release a worktree back to the pool
|
||||||
|
* The worktree is marked as available but NOT deleted
|
||||||
|
*/
|
||||||
|
async release(worktreeId: number): Promise<void> {
|
||||||
|
const worktree = this.worktrees.get(worktreeId);
|
||||||
|
if (!worktree) {
|
||||||
|
console.warn(`⚠️ Worktree ${worktreeId} not found in pool`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n📂 Releasing worktree ${worktreeId} back to pool...`);
|
||||||
|
|
||||||
|
// Mark as available
|
||||||
|
worktree.inUse = false;
|
||||||
|
worktree.currentCommit = undefined;
|
||||||
|
|
||||||
|
console.log(`✓ Worktree ${worktreeId} is now available for reuse`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the EE worktree path for a given worktree
|
||||||
|
*/
|
||||||
|
getEEWorktreePath(worktree: WorktreeInfo): string {
|
||||||
|
return `${worktree.path}_private`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get pool statistics
|
||||||
|
*/
|
||||||
|
getStats(): {
|
||||||
|
total: number;
|
||||||
|
inUse: number;
|
||||||
|
available: number;
|
||||||
|
} {
|
||||||
|
const total = this.worktrees.size;
|
||||||
|
const inUse = Array.from(this.worktrees.values()).filter(
|
||||||
|
(w) => w.inUse
|
||||||
|
).length;
|
||||||
|
return {
|
||||||
|
total,
|
||||||
|
inUse,
|
||||||
|
available: total - inUse,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -71,6 +71,12 @@
|
|||||||
children?: import('svelte').Snippet
|
children?: import('svelte').Snippet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const remoteUrlParam = $page.url.searchParams.get('remote_url')
|
||||||
|
if (remoteUrlParam) {
|
||||||
|
document.cookie = `remote_url=${remoteUrlParam}; path=/; secure; samesite=strict`
|
||||||
|
$page.url.searchParams.delete('remote_url')
|
||||||
|
}
|
||||||
|
|
||||||
let { children }: Props = $props()
|
let { children }: Props = $props()
|
||||||
OpenAPI.WITH_CREDENTIALS = true
|
OpenAPI.WITH_CREDENTIALS = true
|
||||||
let menuOpen = $state(false)
|
let menuOpen = $state(false)
|
||||||
|
|||||||
@@ -13,7 +13,15 @@ export async function onRequest(context) {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const url = new URL(request.url);
|
const url = new URL(request.url);
|
||||||
url.hostname = "app.windmill.dev";
|
|
||||||
|
// Check for remote_url cookie
|
||||||
|
const cookies = request.headers.get("cookie") || "";
|
||||||
|
const remoteUrlMatch = cookies.match(/remote_url=([^;]+)/);
|
||||||
|
const targetHostname = remoteUrlMatch
|
||||||
|
? decodeURIComponent(remoteUrlMatch[1])
|
||||||
|
: "app.windmill.dev";
|
||||||
|
|
||||||
|
url.hostname = targetHostname;
|
||||||
const res = await fetch(url.toString(), {
|
const res = await fetch(url.toString(), {
|
||||||
method: request.method,
|
method: request.method,
|
||||||
body: request.body,
|
body: request.body,
|
||||||
@@ -21,14 +29,18 @@ export async function onRequest(context) {
|
|||||||
redirect: "manual",
|
redirect: "manual",
|
||||||
});
|
});
|
||||||
const newResponse = new Response(res.body, res);
|
const newResponse = new Response(res.body, res);
|
||||||
|
|
||||||
|
// Extract domain from target hostname for cookie domain replacement
|
||||||
|
const targetDomain = targetHostname.split(".").slice(-2).join(".");
|
||||||
|
|
||||||
newResponse.headers.set(
|
newResponse.headers.set(
|
||||||
"set-cookie",
|
"set-cookie",
|
||||||
"token" +
|
"token" +
|
||||||
newResponse.headers
|
(newResponse.headers
|
||||||
.get("set-cookie")
|
.get("set-cookie")
|
||||||
?.replace("Domain=windmill.dev;", "")
|
?.replace(`Domain=${targetDomain};`, "")
|
||||||
?.split("token")
|
?.split("token")
|
||||||
.pop() ?? ""
|
?.pop() ?? "")
|
||||||
);
|
);
|
||||||
|
|
||||||
newResponse.headers.set("Cross-Origin-Resource-Policy", "cross-origin");
|
newResponse.headers.set("Cross-Origin-Resource-Policy", "cross-origin");
|
||||||
|
|||||||
Reference in New Issue
Block a user