feat: workspace dependencies (#7124)
* commit raw requirements Signed-off-by: pyranota <pyra@duck.com> * raw requirements Signed-off-by: pyranota <pyra@duck.com> * implement `parse_annotation` Signed-off-by: pyranota <pyra@duck.com> * more progress on wdeps Signed-off-by: pyranota <pyra@duck.com> * more progress Signed-off-by: pyranota <pyra@duck.com> * fixes Signed-off-by: pyranota <pyra@duck.com> * more progress Signed-off-by: pyranota <pyra@duck.com> * fixes Signed-off-by: pyranota <pyra@duck.com> * cli improvements + raw deps Signed-off-by: pyranota <pyra@duck.com> * cleanup Signed-off-by: pyranota <pyra@duck.com> * fix python versions Signed-off-by: pyranota <pyra@duck.com> * progress Signed-off-by: pyranota <pyra@duck.com> * update :) Signed-off-by: pyranota <pyra@duck.com> * add MaybeLock Signed-off-by: pyranota <pyra@duck.com> * go WIP Signed-off-by: pyranota <pyra@duck.com> * fix python ignoring py version from requirements Signed-off-by: pyranota <pyra@duck.com> * optimize php Signed-off-by: pyranota <pyra@duck.com> * require admin to alter Signed-off-by: pyranota <pyra@duck.com> * fix(cli): flow generateLocks raw deps Signed-off-by: pyranota <pyra@duck.com> * progress in checklist Signed-off-by: pyranota <pyra@duck.com> * fix agent workers Signed-off-by: pyranota <pyra@duck.com> * nits Signed-off-by: pyranota <pyra@duck.com> * nits Signed-off-by: pyranota <pyra@duck.com> * nit: remove default features Signed-off-by: pyranota <pyra@duck.com> * oh-wow Signed-off-by: pyranota <pyra@duck.com> * remove dbg! Signed-off-by: pyranota <pyra@duck.com> * nits Signed-off-by: pyranota <pyra@duck.com> * add indexes Signed-off-by: pyranota <pyra@duck.com> * cleanup Signed-off-by: pyranota <pyra@duck.com> * nits Signed-off-by: pyranota <pyra@duck.com> * remove todos Signed-off-by: pyranota <pyra@duck.com> * fix cli Signed-off-by: pyranota <pyra@duck.com> * add debug flag Signed-off-by: pyranota <pyra@duck.com> * cli: remove noise Signed-off-by: pyranota <pyra@duck.com> * fix cli Signed-off-by: pyranota <pyra@duck.com> * remove todos Signed-off-by: pyranota <pyra@duck.com> * trigger deps correctly Signed-off-by: pyranota <pyra@duck.com> * fix frontend Signed-off-by: pyranota <pyra@duck.com> * fix frontend again Signed-off-by: pyranota <pyra@duck.com> * finally fix frontend Signed-off-by: pyranota <pyra@duck.com> * ee repo ref Signed-off-by: pyranota <pyra@duck.com> * fix all Signed-off-by: pyranota <pyra@duck.com> * more fixes... Signed-off-by: pyranota <pyra@duck.com> * remove test Signed-off-by: pyranota <pyra@duck.com> * Update backend-test.yml * comment out legacy test Signed-off-by: pyranota <pyra@duck.com> * fix ci Signed-off-by: pyranota <pyra@duck.com> * fix ci? Signed-off-by: pyranota <pyra@duck.com> * comment out thing Signed-off-by: pyranota <pyra@duck.com> * ignore test Signed-off-by: pyranota <pyra@duck.com> * ci Signed-off-by: pyranota <pyra@duck.com> * base fixture Signed-off-by: pyranota <pyra@duck.com> * fix regression Signed-off-by: pyranota <pyra@duck.com> * fix docs links Signed-off-by: pyranota <pyra@duck.com> * update min version Signed-off-by: pyranota <pyra@duck.com> * simplify * implement cache for get_latest Signed-off-by: pyranota <pyra@duck.com> * move to workspace settings Signed-off-by: pyranota <pyra@duck.com> * sqlx + migration Signed-off-by: pyranota <pyra@duck.com> * more migrations Signed-off-by: pyranota <pyra@duck.com> * use box pin Signed-off-by: pyranota <pyra@duck.com> * nit Signed-off-by: pyranota <pyra@duck.com> --------- Signed-off-by: pyranota <pyra@duck.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
Pyra
@@ -30,7 +30,8 @@ use windmill_common::{
|
||||
},
|
||||
utils::calculate_hash,
|
||||
worker::{
|
||||
copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG,
|
||||
copy_dir_recursively, pad_string, split_python_requirements, write_file, Connection,
|
||||
PyVAlias, PythonAnnotations, WORKER_CONFIG,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -122,13 +123,13 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result,
|
||||
start_child_process, OccupancyMetrics, StreamNotifier,
|
||||
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file,
|
||||
read_result, start_child_process, OccupancyMetrics, StreamNotifier,
|
||||
},
|
||||
handle_child::handle_child,
|
||||
worker_utils::ping_job_status,
|
||||
PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
|
||||
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
|
||||
PyV, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL,
|
||||
PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
|
||||
};
|
||||
use windmill_common::client::AuthedClient;
|
||||
|
||||
@@ -566,7 +567,7 @@ pub async fn handle_python_job(
|
||||
canceled_by,
|
||||
&mut Some(occupancy_metrics),
|
||||
precomputed_agent_info,
|
||||
annotations,
|
||||
annotations.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -838,10 +839,7 @@ mount {{
|
||||
} else {
|
||||
let args = vec!["-u", "-m", "wrapper"];
|
||||
|
||||
let mut python_cmd = build_command_with_isolation(
|
||||
&python_path,
|
||||
&args,
|
||||
);
|
||||
let mut python_cmd = build_command_with_isolation(&python_path, &args);
|
||||
python_cmd
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
@@ -1177,36 +1175,43 @@ async fn handle_python_deps(
|
||||
let (pyv, resolved_lines) = match requirements_o {
|
||||
// Deployed
|
||||
Some(r) => {
|
||||
let rl = split_requirements(r);
|
||||
let rl = split_python_requirements(r);
|
||||
(PyV::parse_from_requirements(&rl), rl)
|
||||
}
|
||||
// Preview
|
||||
None => {
|
||||
let (v, requirements_lines, error_hint) = match conn {
|
||||
Connection::Sql(db) => {
|
||||
let mut version_specifiers = vec![];
|
||||
let (mut version_specifiers, mut locked_v) = (vec![], None);
|
||||
let (r, h) = Box::pin(windmill_parser_py_imports::parse_python_imports(
|
||||
inner_content,
|
||||
w_id,
|
||||
script_path,
|
||||
db,
|
||||
&mut version_specifiers,
|
||||
&mut locked_v,
|
||||
&None,
|
||||
))
|
||||
.await?;
|
||||
|
||||
let v = PyV::resolve(
|
||||
version_specifiers,
|
||||
job_id,
|
||||
w_id,
|
||||
annotations.py_select_latest,
|
||||
Some(conn.clone()),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let v = if let Some(v) = locked_v {
|
||||
v.into()
|
||||
} else {
|
||||
PyV::resolve(
|
||||
version_specifiers,
|
||||
job_id,
|
||||
w_id,
|
||||
annotations.py_select_latest,
|
||||
Some(conn.clone()),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
|
||||
(v, r, h)
|
||||
}
|
||||
|
||||
Connection::Http(_) => match precomputed_agent_info {
|
||||
Some(PrecomputedAgentInfo::Python {
|
||||
requirements,
|
||||
@@ -1237,7 +1242,7 @@ Returned from server: py_version - {:?}, py_version_v2 - {:?}
|
||||
}
|
||||
};
|
||||
|
||||
let r = split_requirements(requirements.unwrap_or_default());
|
||||
let r = split_python_requirements(requirements.unwrap_or_default());
|
||||
let h = None;
|
||||
|
||||
(v, r, h)
|
||||
@@ -2113,15 +2118,6 @@ pub async fn handle_python_reqs(
|
||||
};
|
||||
}
|
||||
|
||||
pub fn split_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
|
||||
requirements
|
||||
.as_ref()
|
||||
.lines()
|
||||
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
|
||||
.map(String::from)
|
||||
.collect()
|
||||
}
|
||||
|
||||
// Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed
|
||||
fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
|
||||
if skip {
|
||||
@@ -2157,7 +2153,7 @@ pub async fn start_worker(
|
||||
killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
client: windmill_common::client::AuthedClient,
|
||||
) -> error::Result<()> {
|
||||
use crate::{PyV, PyVAlias};
|
||||
use crate::PyV;
|
||||
tracing::info!("script path: {}", script_path);
|
||||
|
||||
let mut mem_peak: i32 = 0;
|
||||
@@ -2317,7 +2313,7 @@ for line in sys.stdin:
|
||||
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
|
||||
|
||||
let py_version = if let Some(requirements) = requirements_o {
|
||||
PyV::parse_from_requirements(&split_requirements(requirements.as_str()))
|
||||
PyV::parse_from_requirements(&split_python_requirements(requirements.as_str()))
|
||||
} else {
|
||||
tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11");
|
||||
PyVAlias::Py311.into()
|
||||
|
||||
Reference in New Issue
Block a user