diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 2aa636ba99..b09c0ee04d 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -55,13 +55,25 @@ jobs: - name: Docker meta - id: meta-slim-public + id: meta-public if: github.event_name != 'pull_request' uses: docker/metadata-action@v4 with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Docker meta + id: meta-ee-public + if: github.event_name != 'pull_request' + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee tags: | type=ref,event=branch type=ref,event=pr @@ -85,13 +97,30 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:latest - ${{ steps.meta-slim-public.outputs.tags }} + ${{ steps.meta-public.outputs.tags }} labels: | - ${{ steps.meta-slim-public.outputs.labels }} + ${{ steps.meta-public.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache + - name: Build and push publicly ee + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v3 + with: + context: . + push: true + build-args: | + features=enterprise + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest + ${{ steps.meta-ee-public.outputs.tags }} + labels: | + ${{ steps.meta-ee-public.outputs.labels }} + org.opencontainers.image.licenses=Windmill-Enterprise-License + cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache + cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache + playwright: runs-on: [self-hosted, new] @@ -125,6 +154,7 @@ jobs: run: docker kill ${{ steps.docker-container.outputs.id }} if: always() + publish_privately_heavy: runs-on: [self-hosted, new] if: github.event_name != 'pull_request' @@ -163,9 +193,7 @@ jobs: with: context: . push: true - file: ./Dockerfile - build-args: | - features=enterprise + file: ./docker/DockerfileHeavy tags: | ${{ steps.meta-heavy.outputs.tags }} labels: ${{ steps.meta-heavy.outputs.labels }} diff --git a/backend/src/main.rs b/backend/src/main.rs index 8872fcc5d4..f109da0fb2 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -87,6 +87,7 @@ async fn main() -> anyhow::Result<()> { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(false); + let license_key = std::env::var("LICENSE_KEY").ok(); let sync_bucket = std::env::var("S3_CACHE_BUCKET") .ok() .map(|e| Some(e)) @@ -112,6 +113,7 @@ async fn main() -> anyhow::Result<()> { }, rx.resubscribe(), sync_bucket, + license_key, ) .await?; } @@ -166,7 +168,22 @@ pub async fn run_workers( worker_config: WorkerConfig, rx: tokio::sync::broadcast::Receiver<()>, mut periodic_script: Option, + license_key: Option, ) -> anyhow::Result<()> { + #[cfg(feature = "enterprise")] + if let Some(license_key) = license_key { + if license_key != "REQUIRED_DEC1" { + panic!("Invalid license key"); + } + } else { + panic!("License key is required for the enterprise edition"); + } + + #[cfg(not(feature = "enterprise"))] + if license_key.is_some() { + panic!("License key is required ONLY for the enterprise edition"); + } + let instance_name = rd_string(5); let monitor = tokio_metrics::TaskMonitor::new(); diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index bdf96397ad..82dc08f5ea 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -86,33 +86,17 @@ mount { } -mount { - src: "{JOB_DIR}/requirements.txt" - dst: "/user/requirements.txt" - is_bind: true -} - -mount { - src: "{JOB_DIR}/dependencies" - dst: "/out" - is_bind: true - rw: true -} - - mount { src: "{WORKER_DIR}/download_deps.py.sh" dst: "/download_deps.sh" is_bind: true } - mount { src: "{CACHE_DIR}" - dst: "/tmp/.cache/pip" + dst: "{CACHE_DIR}" is_bind: true rw: true - mandatory: false } exec_bin { diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index d77849d462..a17c3d7665 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,9 +19,6 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -mkdir -p /tmp/dependencies -touch /tmp/dependencies/_windmill -/usr/local/bin/python3 -m pip install --cache-dir /tmp/.cache/pip -t /tmp/dependencies -r /user/requirements.txt\ +/usr/local/bin/python3 -m pip install $REQ -I -t $TARGET --no-cache\ --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG -mv /tmp/dependencies/* /out diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 0efea2bd64..b27037d8ee 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -91,13 +91,6 @@ mount { is_bind: true } - -mount { - src: "{JOB_DIR}/dependencies" - dst: "/tmp/dependencies" - is_bind: true -} - mount { src: "/etc/ssl" dst: "/etc/ssl" @@ -130,7 +123,7 @@ mount { iface_no_lo: true envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" -envar: "PYTHONPATH=/tmp/dependencies{ADDITIONAL_PYTHON_PATHS}" +envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}" envar: "HOME=/tmp" diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a241788e80..39809f8004 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -155,33 +155,12 @@ pub async fn create_token_for_owner<'c>( } const TMP_DIR: &str = "/tmp/windmill"; -const PIP_SUPERCACHE_DIR: &str = "/tmp/windmill/cache/pip_permanent"; const ROOT_CACHE_DIR: &str = "/tmp/windmill/cache/"; const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); const NUM_SECS_ENV_CHECK: u64 = 15; const NUM_SECS_SYNC: u64 = 60 * 10; -const DEFAULT_HEAVY_DEPS: [&str; 18] = [ - "numpy", - "pandas", - "anyio", - "attrs", - "certifi", - "h11", - "httpcore", - "httpx", - "idna", - "python-dateutil", - "rfc3986", - "six", - "sniffio", - "windmill-api", - "wmill", - "psycopg2-binary", - "matplotlib", - "seaborn", -]; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); @@ -248,13 +227,7 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir"); - for x in [ - &worker_dir, - PIP_SUPERCACHE_DIR, - PIP_CACHE_DIR, - DENO_CACHE_DIR, - GO_CACHE_DIR, - ] { + for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR, GO_CACHE_DIR] { DirBuilder::new() .recursive(true) .create(x) @@ -316,9 +289,6 @@ pub async fn run_worker( let go_path = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string()); let python_path = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); - let python_heavy_deps = std::env::var("PYTHON_HEAVY_DEPS") - .map(|x| x.split(',').map(|x| x.to_string()).collect::>()) - .unwrap_or_else(|_| vec![]); let nsjail_path = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); let path_env = std::env::var("PATH").unwrap_or_else(|_| String::new()); let home_env = std::env::var("HOME").unwrap_or_else(|_| String::new()); @@ -329,7 +299,6 @@ pub async fn run_worker( deno_path, go_path, python_path, - python_heavy_deps, nsjail_path, path_env, home_env, @@ -613,7 +582,6 @@ struct Envs { deno_path: String, go_path: String, python_path: String, - python_heavy_deps: Vec, nsjail_path: String, path_env: String, home_env: String, @@ -1435,7 +1403,6 @@ async fn handle_python_job( envs @ Envs { nsjail_path, python_path, - python_heavy_deps, path_env, pip_extra_index_url, pip_index_url, @@ -1481,7 +1448,6 @@ async fn handle_python_job( job_dir, "download.config.proto", &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - .replace("{JOB_DIR}", job_dir) .replace("{WORKER_DIR}", &worker_dir) .replace("{CACHE_DIR}", PIP_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), @@ -1489,18 +1455,6 @@ async fn handle_python_job( .await?; } - let mut heavy_deps = DEFAULT_HEAVY_DEPS - .iter() - .map(|s| s.to_string()) - .collect::>(); - heavy_deps.extend(python_heavy_deps.into_iter().map(|s| s.to_string())); - - let (heavy, regular): (Vec<&str>, Vec<&str>) = requirements - .split("\n") - .partition(|d| heavy_deps.iter().any(|hd| d.starts_with(hd))); - - let _ = write_file(job_dir, "requirements.txt", ®ular.join("\n")).await?; - let mut vars = vec![("PATH", path_env)]; if let Some(url) = pip_extra_index_url { vars.push(("EXTRA_INDEX_URL", url)); @@ -1512,80 +1466,20 @@ async fn handle_python_job( vars.push(("TRUSTED_HOST", host)); } - if heavy.len() > 0 { - logs.push_str(&format!( - "\nheavy deps detected, using supercache for: {heavy:?}" - )); - additional_python_paths = - handle_python_heavy_reqs(python_path, heavy, vars.clone(), job, logs, db, timeout) - .await?; - } - - if regular.len() > 0 { - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started setup python dependencies" - ); - - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(vars) - .args(vec!["--config", "download.config.proto"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - let mut args = vec![ - "-m", - "pip", - "install", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - "./dependencies", - "-r", - "./requirements.txt", - ]; - if let Some(url) = pip_extra_index_url { - args.extend(["--extra-index-url", url]); - } - if let Some(url) = pip_index_url { - args.extend(["--index-url", url]); - } - if let Some(host) = pip_trusted_host { - args.extend(["--trusted-host", host]); - } - Command::new(python_path) - .current_dir(job_dir) - .env_clear() - .envs(vars) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - - logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); - let child = handle_child(&job.id, db, logs, timeout, child).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = child.is_ok(), - "finished setting up python dependencies {}", - job.id - ); - child?; - } else { - logs.push_str("\nskipping pip install since not needed"); - }; + additional_python_paths = handle_python_reqs( + python_path, + requirements.split("\n").collect(), + vars.clone(), + job, + logs, + db, + timeout, + nsjail_path, + disable_nsjail.clone(), + worker_name, + job_dir, + ) + .await?; } logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n"); @@ -1642,10 +1536,7 @@ with open("result.json", 'w') as f: write_file(job_dir, "main.py", &wrapper_content).await?; let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?; - let additional_python_paths_folders = additional_python_paths - .iter() - .map(|x| format!(":{x}")) - .join(""); + let additional_python_paths_folders = additional_python_paths.iter().join(":"); if !disable_nsjail { let shared_deps = additional_python_paths .into_iter() @@ -1677,10 +1568,7 @@ mount {{ ) .await?; } else { - reserved_variables.insert( - "PYTHONPATH".to_string(), - format!("{job_dir}/dependencies{additional_python_paths_folders}"), - ); + reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); } tracing::info!( @@ -1972,7 +1860,7 @@ async fn pip_compile( db: &Pool, timeout: i32, ) -> error::Result { - logs.push_str(&format!("content of requirements:\n{}\n", requirements)); + logs.push_str(&format!("\ncontent of requirements:\n{}", requirements)); let file = "requirements.in"; write_file(job_dir, file, &requirements).await?; let mut args = vec!["-q", "--no-header", file]; @@ -2537,50 +2425,86 @@ async fn handle_zombie_jobs(db: &Pool, timeout: i32, base_url: &str) { } } -async fn handle_python_heavy_reqs( +async fn handle_python_reqs( python_path: &String, - heavy_requirements: Vec<&str>, + requirements: Vec<&str>, vars: Vec<(&str, &String)>, job: &QueuedJob, logs: &mut String, db: &sqlx::Pool, timeout: i32, + nsjail_path: &str, + disable_nsjail: bool, + worker_name: &str, + job_dir: &str, ) -> error::Result> { let mut req_paths: Vec = vec![]; - for req in heavy_requirements { + for req in requirements { // todo: handle many reqs - let venv_p = format!("{PIP_SUPERCACHE_DIR}/{req}"); + let venv_p = format!("{PIP_CACHE_DIR}/{req}"); if metadata(&venv_p).await.is_ok() { tracing::info!("already exists: {:?}", &venv_p); req_paths.push(venv_p); continue; } - logs.push_str("\n--- PIP SUPERCACHE INSTALL ---\n"); - logs.push_str(&format!("\nthe heavy dependency {req} is being installed for the first time.\nIt will take a bit longer but further execution will be much faster!")); + logs.push_str("\n--- PIP INSTALL ---\n"); + logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses.")); - logs.push_str("pip install\n"); - let child = Command::new(python_path) - .env_clear() - .envs(vars.clone()) - .args(vec![ - "-m", - "pip", - "install", - &req, - "-I", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - venv_p.as_str(), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - handle_child(&job.id, db, logs, timeout, child).await?; + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started setup python dependencies" + ); + + let child = if !disable_nsjail { + let mut vars = vars.clone(); + let req = req.to_string(); + vars.push(("REQ", &req)); + vars.push(("TARGET", &venv_p)); + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(vars) + .args(vec!["--config", "download.config.proto"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(python_path) + .env_clear() + .envs(vars.clone()) + .args(vec![ + "-m", + "pip", + "install", + &req, + "-I", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + venv_p.as_str(), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + + logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); + let child = handle_child(&job.id, db, logs, timeout, child).await; + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = child.is_ok(), + "finished setting up python dependencies {}", + job.id + ); + child?; req_paths.push(venv_p); } diff --git a/docker/DockerfileHeavy b/docker/DockerfileHeavy index 6740ab9ae6..9170330d9f 100644 --- a/docker/DockerfileHeavy +++ b/docker/DockerfileHeavy @@ -1,4 +1,4 @@ -FROM ghcr.io/windmill-labs/windmill +FROM ghcr.io/windmill-labs/windmill-ee RUN /usr/local/bin/python3 -m pip install nltk RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon