kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backend): global cache refactor for pip using tar for each dependency (#1443)

Browse Source 
* cache refactor

* exclude tar from being synced to bucket

* run

* update

* update
This commit is contained in:
Ruben Fiszel
2023-04-20 20:05:12 +02:00
committed by GitHub
parent 3c98452f50
commit 369dd0dac6
5 changed files with 284 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/windmill-worker/src/python_executor.rs
View File

@@ -45,11 +45,14 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
#[cfg(feature = "enterprise")]
use crate::global_cache::{build_tar_and_push, pull_from_tar};
use crate::{
common::{read_result, set_logs},
create_args_and_out_file, get_reserved_variables, handle_child, write_file,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
PIP_CACHE_DIR,
PIP_CACHE_DIR, S3_CACHE_BUCKET,
};
pub async fn create_dependencies_dir(job_dir: &str) {
@@ -465,6 +468,14 @@ pub async fn handle_python_reqs(
continue;
}
#[cfg(feature = "enterprise")]
if let Some(ref bucket) = *S3_CACHE_BUCKET {
if pull_from_tar(bucket, venv_p.clone()).await.is_ok() {
req_paths.push(venv_p.clone());
continue;
}
}
logs.push_str("\n--- PIP INSTALL ---\n");
logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
@@ -548,6 +559,11 @@ pub async fn handle_python_reqs(
);
child?;
#[cfg(feature = "enterprise")]
if let Some(ref bucket) = *S3_CACHE_BUCKET {
let venv_p = venv_p.clone();
tokio::spawn(build_tar_and_push(bucket, venv_p));
}
req_paths.push(venv_p);
}
Ok(req_paths)