kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: supercache extended to all version

Browse Source
This commit is contained in:
Ruben Fiszel
2022-11-18 20:17:23 +01:00
parent 7044dc7eb1
commit 0a8fd771b0
7 changed files with 138 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
.github/workflows/docker-image.yml vendored
View File

@@ -55,13 +55,25 @@ jobs:
- name: Docker meta
id: meta-slim-public
id: meta-public
if: github.event_name != 'pull_request'
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Docker meta
id: meta-ee-public
if: github.event_name != 'pull_request'
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
tags: |
type=ref,event=branch
type=ref,event=pr
@@ -85,13 +97,30 @@ jobs:
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:latest
${{ steps.meta-slim-public.outputs.tags }}
${{ steps.meta-public.outputs.tags }}
labels: |
${{ steps.meta-slim-public.outputs.labels }}
${{ steps.meta-public.outputs.labels }}
org.opencontainers.image.licenses=AGPLv3
cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-slim:buildcache
- name: Build and push publicly ee
if: github.event_name != 'pull_request'
uses: docker/build-push-action@v3
with:
context: .
push: true
build-args: |
features=enterprise
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
${{ steps.meta-ee-public.outputs.tags }}
labels: |
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache
cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:buildcache
playwright:
runs-on: [self-hosted, new]
@@ -125,6 +154,7 @@ jobs:
run: docker kill ${{ steps.docker-container.outputs.id }}
if: always()
publish_privately_heavy:
runs-on: [self-hosted, new]
if: github.event_name != 'pull_request'
@@ -163,9 +193,7 @@ jobs:
with:
context: .
push: true
file: ./Dockerfile
build-args: |
features=enterprise
file: ./docker/DockerfileHeavy
tags: |
${{ steps.meta-heavy.outputs.tags }}
labels: ${{ steps.meta-heavy.outputs.labels }}

17
backend/src/main.rs
View File

@@ -87,6 +87,7 @@ async fn main() -> anyhow::Result<()> {
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let license_key = std::env::var("LICENSE_KEY").ok();
let sync_bucket = std::env::var("S3_CACHE_BUCKET")
.ok()
.map(|e| Some(e))
@@ -112,6 +113,7 @@ async fn main() -> anyhow::Result<()> {
},
rx.resubscribe(),
sync_bucket,
license_key,
)
.await?;
}
@@ -166,7 +168,22 @@ pub async fn run_workers(
worker_config: WorkerConfig,
rx: tokio::sync::broadcast::Receiver<()>,
mut periodic_script: Option<String>,
license_key: Option<String>,
) -> anyhow::Result<()> {
#[cfg(feature = "enterprise")]
if let Some(license_key) = license_key {
if license_key != "REQUIRED_DEC1" {
panic!("Invalid license key");
}
} else {
panic!("License key is required for the enterprise edition");
}
#[cfg(not(feature = "enterprise"))]
if license_key.is_some() {
panic!("License key is required ONLY for the enterprise edition");
}
let instance_name = rd_string(5);
let monitor = tokio_metrics::TaskMonitor::new();

18
backend/windmill-worker/nsjail/download.py.config.proto
View File

@@ -86,33 +86,17 @@ mount {
}
mount {
src: "{JOB_DIR}/requirements.txt"
dst: "/user/requirements.txt"
is_bind: true
}
mount {
src: "{JOB_DIR}/dependencies"
dst: "/out"
is_bind: true
rw: true
}
mount {
src: "{WORKER_DIR}/download_deps.py.sh"
dst: "/download_deps.sh"
is_bind: true
}
mount {
src: "{CACHE_DIR}"
dst: "/tmp/.cache/pip"
dst: "{CACHE_DIR}"
is_bind: true
rw: true
mandatory: false
}
exec_bin {

5
backend/windmill-worker/nsjail/download_deps.py.sh
View File

@@ -19,9 +19,6 @@ then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
mkdir -p /tmp/dependencies
touch /tmp/dependencies/_windmill
/usr/local/bin/python3 -m pip install --cache-dir /tmp/.cache/pip -t /tmp/dependencies -r /user/requirements.txt\
/usr/local/bin/python3 -m pip install $REQ -I -t $TARGET --no-cache\
--no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
mv /tmp/dependencies/* /out

9
backend/windmill-worker/nsjail/run.python3.config.proto
View File

@@ -91,13 +91,6 @@ mount {
is_bind: true
}
mount {
src: "{JOB_DIR}/dependencies"
dst: "/tmp/dependencies"
is_bind: true
}
mount {
src: "/etc/ssl"
dst: "/etc/ssl"
@@ -130,7 +123,7 @@ mount {
iface_no_lo: true
envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
envar: "PYTHONPATH=/tmp/dependencies{ADDITIONAL_PYTHON_PATHS}"
envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}"
envar: "HOME=/tmp"

240
backend/windmill-worker/src/worker.rs
View File

@@ -155,33 +155,12 @@ pub async fn create_token_for_owner<'c>(
}
const TMP_DIR: &str = "/tmp/windmill";
const PIP_SUPERCACHE_DIR: &str = "/tmp/windmill/cache/pip_permanent";
const ROOT_CACHE_DIR: &str = "/tmp/windmill/cache/";
const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go");
const NUM_SECS_ENV_CHECK: u64 = 15;
const NUM_SECS_SYNC: u64 = 60 * 10;
const DEFAULT_HEAVY_DEPS: [&str; 18] = [
"numpy",
"pandas",
"anyio",
"attrs",
"certifi",
"h11",
"httpcore",
"httpx",
"idna",
"python-dateutil",
"rfc3986",
"six",
"sniffio",
"windmill-api",
"wmill",
"psycopg2-binary",
"matplotlib",
"seaborn",
];
const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
@@ -248,13 +227,7 @@ pub async fn run_worker(
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
for x in [
&worker_dir,
PIP_SUPERCACHE_DIR,
PIP_CACHE_DIR,
DENO_CACHE_DIR,
GO_CACHE_DIR,
] {
for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR, GO_CACHE_DIR] {
DirBuilder::new()
.recursive(true)
.create(x)
@@ -316,9 +289,6 @@ pub async fn run_worker(
let go_path = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
let python_path =
std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
let python_heavy_deps = std::env::var("PYTHON_HEAVY_DEPS")
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>())
.unwrap_or_else(|_| vec![]);
let nsjail_path = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string());
let path_env = std::env::var("PATH").unwrap_or_else(|_| String::new());
let home_env = std::env::var("HOME").unwrap_or_else(|_| String::new());
@@ -329,7 +299,6 @@ pub async fn run_worker(
deno_path,
go_path,
python_path,
python_heavy_deps,
nsjail_path,
path_env,
home_env,
@@ -613,7 +582,6 @@ struct Envs {
deno_path: String,
go_path: String,
python_path: String,
python_heavy_deps: Vec<String>,
nsjail_path: String,
path_env: String,
home_env: String,
@@ -1435,7 +1403,6 @@ async fn handle_python_job(
envs @ Envs {
nsjail_path,
python_path,
python_heavy_deps,
path_env,
pip_extra_index_url,
pip_index_url,
@@ -1481,7 +1448,6 @@ async fn handle_python_job(
job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR)
.replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()),
@@ -1489,18 +1455,6 @@ async fn handle_python_job(
.await?;
}
let mut heavy_deps = DEFAULT_HEAVY_DEPS
.iter()
.map(|s| s.to_string())
.collect::<Vec<String>>();
heavy_deps.extend(python_heavy_deps.into_iter().map(|s| s.to_string()));
let (heavy, regular): (Vec<&str>, Vec<&str>) = requirements
.split("\n")
.partition(|d| heavy_deps.iter().any(|hd| d.starts_with(hd)));
let _ = write_file(job_dir, "requirements.txt", &regular.join("\n")).await?;
let mut vars = vec![("PATH", path_env)];
if let Some(url) = pip_extra_index_url {
vars.push(("EXTRA_INDEX_URL", url));
@@ -1512,80 +1466,20 @@ async fn handle_python_job(
vars.push(("TRUSTED_HOST", host));
}
if heavy.len() > 0 {
logs.push_str(&format!(
"\nheavy deps detected, using supercache for: {heavy:?}"
));
additional_python_paths =
handle_python_heavy_reqs(python_path, heavy, vars.clone(), job, logs, db, timeout)
.await?;
}
if regular.len() > 0 {
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
"started setup python dependencies"
);
let child = if !disable_nsjail {
Command::new(nsjail_path)
.current_dir(job_dir)
.env_clear()
.envs(vars)
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
} else {
let mut args = vec![
"-m",
"pip",
"install",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
"./dependencies",
"-r",
"./requirements.txt",
];
if let Some(url) = pip_extra_index_url {
args.extend(["--extra-index-url", url]);
}
if let Some(url) = pip_index_url {
args.extend(["--index-url", url]);
}
if let Some(host) = pip_trusted_host {
args.extend(["--trusted-host", host]);
}
Command::new(python_path)
.current_dir(job_dir)
.env_clear()
.envs(vars)
.args(args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
};
logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n");
let child = handle_child(&job.id, db, logs, timeout, child).await;
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
is_ok = child.is_ok(),
"finished setting up python dependencies {}",
job.id
);
child?;
} else {
logs.push_str("\nskipping pip install since not needed");
};
additional_python_paths = handle_python_reqs(
python_path,
requirements.split("\n").collect(),
vars.clone(),
job,
logs,
db,
timeout,
nsjail_path,
disable_nsjail.clone(),
worker_name,
job_dir,
)
.await?;
}
logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
@@ -1642,10 +1536,7 @@ with open("result.json", 'w') as f:
write_file(job_dir, "main.py", &wrapper_content).await?;
let mut reserved_variables = get_reserved_variables(job, &token, &base_url, db).await?;
let additional_python_paths_folders = additional_python_paths
.iter()
.map(|x| format!(":{x}"))
.join("");
let additional_python_paths_folders = additional_python_paths.iter().join(":");
if !disable_nsjail {
let shared_deps = additional_python_paths
.into_iter()
@@ -1677,10 +1568,7 @@ mount {{
)
.await?;
} else {
reserved_variables.insert(
"PYTHONPATH".to_string(),
format!("{job_dir}/dependencies{additional_python_paths_folders}"),
);
reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
}
tracing::info!(
@@ -1972,7 +1860,7 @@ async fn pip_compile(
db: &Pool<Postgres>,
timeout: i32,
) -> error::Result<String> {
logs.push_str(&format!("content of requirements:\n{}\n", requirements));
logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
let file = "requirements.in";
write_file(job_dir, file, &requirements).await?;
let mut args = vec!["-q", "--no-header", file];
@@ -2537,50 +2425,86 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, timeout: i32, base_url: &str) {
}
}
async fn handle_python_heavy_reqs(
async fn handle_python_reqs(
python_path: &String,
heavy_requirements: Vec<&str>,
requirements: Vec<&str>,
vars: Vec<(&str, &String)>,
job: &QueuedJob,
logs: &mut String,
db: &sqlx::Pool<sqlx::Postgres>,
timeout: i32,
nsjail_path: &str,
disable_nsjail: bool,
worker_name: &str,
job_dir: &str,
) -> error::Result<Vec<String>> {
let mut req_paths: Vec<String> = vec![];
for req in heavy_requirements {
for req in requirements {
// todo: handle many reqs
let venv_p = format!("{PIP_SUPERCACHE_DIR}/{req}");
let venv_p = format!("{PIP_CACHE_DIR}/{req}");
if metadata(&venv_p).await.is_ok() {
tracing::info!("already exists: {:?}", &venv_p);
req_paths.push(venv_p);
continue;
}
logs.push_str("\n--- PIP SUPERCACHE INSTALL ---\n");
logs.push_str(&format!("\nthe heavy dependency {req} is being installed for the first time.\nIt will take a bit longer but further execution will be much faster!"));
logs.push_str("\n--- PIP INSTALL ---\n");
logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
logs.push_str("pip install\n");
let child = Command::new(python_path)
.env_clear()
.envs(vars.clone())
.args(vec![
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
handle_child(&job.id, db, logs, timeout, child).await?;
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
"started setup python dependencies"
);
let child = if !disable_nsjail {
let mut vars = vars.clone();
let req = req.to_string();
vars.push(("REQ", &req));
vars.push(("TARGET", &venv_p));
Command::new(nsjail_path)
.current_dir(job_dir)
.env_clear()
.envs(vars)
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
} else {
Command::new(python_path)
.env_clear()
.envs(vars.clone())
.args(vec![
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
};
logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n");
let child = handle_child(&job.id, db, logs, timeout, child).await;
tracing::info!(
worker_name = %worker_name,
job_id = %job.id,
workspace_id = %job.workspace_id,
is_ok = child.is_ok(),
"finished setting up python dependencies {}",
job.id
);
child?;
req_paths.push(venv_p);
}

2
docker/DockerfileHeavy
View File

@@ -1,4 +1,4 @@
FROM ghcr.io/windmill-labs/windmill
FROM ghcr.io/windmill-labs/windmill-ee
RUN /usr/local/bin/python3 -m pip install nltk
RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon