kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: parse script for preprocessor/no_main_func on deploy (#5292)

Browse Source 
* feat: parse script for preprocessor/no_main_func on deploy

* fix substitute for linux
This commit is contained in:
HugoCasa
2025-02-14 15:42:18 +01:00
committed by GitHub
parent d3a4e39bc0
commit 88ae1dc990
13 changed files with 85 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/Cargo.lock generated
View File

@@ -10988,6 +10988,7 @@ dependencies = [
"windmill-git-sync",
"windmill-indexer",
"windmill-parser",
"windmill-parser-py",
"windmill-parser-ts",
"windmill-queue",
]

27
backend/parsers/windmill-parser-py/src/lib.rs
View File

@@ -60,6 +60,7 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
pub fn parse_python_signature(
code: &str,
override_main: Option<String>,
skip_params: bool,
) -> anyhow::Result<MainArgSignature> {
let main_name = override_main.unwrap_or("main".to_string());
@@ -78,11 +79,13 @@ pub fn parse_python_signature(
let ast = Suite::parse(&filtered_code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let param = ast.into_iter().find_map(|x| match x {
let params = ast.into_iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => Some(*args),
_ => None,
});
if let Some(params) = param {
if !skip_params && params.is_some() {
let params = params.unwrap();
//println!("{:?}", params);
let def_arg_start = params.args.len() - params.defaults().count();
Ok(MainArgSignature {
@@ -149,7 +152,7 @@ pub fn parse_python_signature(
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
no_main_func: Some(params.is_none()),
has_preprocessor: Some(has_preprocessor),
})
}
@@ -287,7 +290,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
";
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -376,7 +379,7 @@ def main(test1: str,
";
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -436,7 +439,7 @@ def main(test1: str,
";
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -493,7 +496,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -537,7 +540,7 @@ def main(test1: DynSelect_foo): return
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -568,7 +571,7 @@ def hello(): return
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -596,7 +599,7 @@ def main(): return
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -617,10 +620,10 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b
"#;
println!(
"{}",
serde_json::to_string(&parse_python_signature(code, None)?)?
serde_json::to_string(&parse_python_signature(code, None, false)?)?
);
assert_eq!(
parse_python_signature(code, None)?,
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,

42
backend/parsers/windmill-parser-ts/src/lib.rs
View File

@@ -134,6 +134,7 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
pub fn parse_deno_signature(
code: &str,
skip_dflt: bool,
skip_params: bool,
main_override: Option<String>,
) -> anyhow::Result<MainArgSignature> {
let cm: Lrc<SourceMap> = Default::default();
@@ -179,27 +180,26 @@ pub fn parse_deno_signature(
});
let mut c: u16 = 0;
if let Some(params) = params {
let r = MainArgSignature {
star_args: false,
star_kwargs: false,
args: params
.into_iter()
.map(|x| parse_param(x, &cm, skip_dflt, &mut c))
.collect::<anyhow::Result<Vec<Arg>>>()?,
no_main_func: Some(false),
has_preprocessor: Some(has_preprocessor),
};
Ok(r)
} else {
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: Some(has_preprocessor),
})
}
let no_main_func = params.is_none();
let r = MainArgSignature {
star_args: false,
star_kwargs: false,
args: if skip_params {
vec![]
} else {
params
.map(|x| {
x.into_iter()
.map(|x| parse_param(x, &cm, skip_dflt, &mut c))
.collect::<anyhow::Result<Vec<Arg>>>()
})
.transpose()?
.unwrap_or_else(|| vec![])
},
no_main_func: Some(no_main_func),
has_preprocessor: Some(has_preprocessor),
};
Ok(r)
}
fn parse_param(

4
backend/parsers/windmill-parser-wasm/src/lib.rs
View File

@@ -17,10 +17,11 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
pub fn parse_deno(code: &str, main_override: Option<String>, skip_params: Option<bool>) -> String {
wrap_sig(windmill_parser_ts::parse_deno_signature(
code,
false,
false,
main_override,
))
}
@@ -73,6 +74,7 @@ pub fn parse_python(code: &str, main_override: Option<String>) -> String {
wrap_sig(windmill_parser_py::parse_python_signature(
code,
main_override,
false,
))
}

8
backend/parsers/windmill-parser-wasm/tests/wasm.rs
View File

@@ -18,7 +18,7 @@ export function main(test1?: string, test2: string = \"burkina\",
}
";
assert_eq!(
parse_deno_signature(code, false, None)?,
parse_deno_signature(code, false, false, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -159,7 +159,7 @@ export function main(test2 = \"burkina\",
}
";
assert_eq!(
parse_deno_signature(code, false, None)?,
parse_deno_signature(code, false, false, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -236,7 +236,7 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
}
";
assert_eq!(
parse_deno_signature(code, false, None)?,
parse_deno_signature(code, false, false, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
@@ -282,7 +282,7 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
}
";
assert_eq!(
parse_deno_signature(code, false, None)?,
parse_deno_signature(code, false, false, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,

4
backend/substitute_ee_code.sh
View File

@@ -68,7 +68,7 @@ fi
if [ "$REVERT" == "YES" ]; then
for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${ee_file/${EE_CODE_DIR}/}"
ce_file="${root_dirpath}/backend/${ce_file}"
if [ "$REVERT_PREVIOUS" == "YES" ]; then
git checkout HEAD@{3} ${ce_file} || true
@@ -80,7 +80,7 @@ if [ "$REVERT" == "YES" ]; then
else
# This replaces all files in current repo with alternative EE files in windmill-ee-private
for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${ee_file/${EE_CODE_DIR}/}"
ce_file="${root_dirpath}/backend/${ce_file}"
if [[ -f "${ce_file}" ]]; then
rm "${ce_file}"

1
backend/windmill-api/Cargo.toml
View File

@@ -36,6 +36,7 @@ windmill-common = { workspace = true, default-features = false }
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-py.workspace = true
windmill-git-sync.workspace = true
windmill-indexer = { workspace = true, optional = true }
tokio.workspace = true

17
backend/windmill-api/src/scripts.rs
View File

@@ -625,6 +625,19 @@ async fn create_script_internal<'c>(
} else {
ns.language.clone()
};
let (no_main_func, has_preprocessor) = match lang {
ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => {
let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None)?;
(args.no_main_func, args.has_preprocessor)
}
ScriptLang::Python3 => {
let args = windmill_parser_py::parse_python_signature(&ns.content, None, true)?;
(args.no_main_func, args.has_preprocessor)
}
_ => (ns.no_main_func, ns.has_preprocessor),
};
sqlx::query!(
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
@@ -660,9 +673,9 @@ async fn create_script_internal<'c>(
ns.timeout,
ns.concurrency_key,
ns.visible_to_runner_only,
ns.no_main_func,
no_main_func.filter(|x| *x), // should be Some(true) or None
codebase,
ns.has_preprocessor,
has_preprocessor.filter(|x| *x), // should be Some(true) or None
if ns.on_behalf_of_email.is_some() {
Some(&authed.email)
} else {

4
backend/windmill-worker/src/bun_executor.rs
View File

@@ -951,6 +951,7 @@ pub async fn handle_bun_job(
let args = windmill_parser_ts::parse_deno_signature(
inner_content,
true,
false,
main_override.map(ToString::to_string),
)?
.args;
@@ -960,6 +961,7 @@ pub async fn handle_bun_job(
windmill_parser_ts::parse_deno_signature(
inner_content,
true,
false,
Some("preprocessor".to_string()),
)?
.args,
@@ -1573,7 +1575,7 @@ pub async fn start_worker(
{
// let mut start = Instant::now();
let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args;
let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args;
let dates = args
.iter()
.filter_map(|x| {

4
backend/windmill-worker/src/deno_executor.rs
View File

@@ -205,6 +205,7 @@ pub async fn handle_deno_job(
let args = windmill_parser_ts::parse_deno_signature(
inner_content,
true,
false,
main_override.map(ToString::to_string),
)?
.args;
@@ -214,6 +215,7 @@ pub async fn handle_deno_job(
windmill_parser_ts::parse_deno_signature(
inner_content,
true,
false,
Some("preprocessor".to_string()),
)?
.args,
@@ -533,7 +535,7 @@ pub async fn start_worker(
{
// let mut start = Instant::now();
let args = windmill_parser_ts::parse_deno_signature(inner_content, true, None)?.args;
let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args;
let dates = args
.iter()
.filter_map(|x| {

2
backend/windmill-worker/src/js_eval.rs
View File

@@ -775,7 +775,7 @@ pub async fn eval_fetch_timeout(
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, None)?.args;
let parsed_args = windmill_parser_ts::parse_deno_signature(&ts_expr, true, false, None)?.args;
let spread = parsed_args
.into_iter()
.map(|x| {

4
backend/windmill-worker/src/python_executor.rs
View File

@@ -1296,12 +1296,14 @@ async fn prepare_wrapper(
let sig = windmill_parser_py::parse_python_signature(
inner_content,
main_override.map(ToString::to_string),
false,
)?;
let pre_sig = if apply_preprocessor {
Some(windmill_parser_py::parse_python_signature(
inner_content,
Some("preprocessor".to_string()),
false,
)?)
} else {
None
@@ -1661,7 +1663,7 @@ async fn spawn_uv_install(
.replace("{TARGET_DIR}", &venv_p)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
nsjail_cmd
.current_dir(job_dir)

16
cli/metadata.ts
View File

@@ -313,10 +313,16 @@ export async function updateScriptSchema(
path
);
metadataContent.schema = result.schema;
if (result.has_preprocessor == true)
if (result.has_preprocessor) {
metadataContent.has_preprocessor = result.has_preprocessor;
if (result.no_main_func === true)
} else {
delete metadataContent.has_preprocessor;
}
if (result.no_main_func) {
metadataContent.no_main_func = result.no_main_func;
} else {
delete metadataContent.no_main_func;
}
}
async function updateScriptLock(
@@ -441,7 +447,11 @@ export function inferSchema(
content: string,
currentSchema: any,
path: string
) {
): {
schema: any;
has_preprocessor: boolean | undefined;
no_main_func: boolean | undefined;
} {
let inferedSchema: any;
if (language === "python3") {
inferedSchema = JSON.parse(parse_python(content));