From ea9487e3cfd3e28a54471c82e6e96bb99f52ab62 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 3 Jul 2025 23:34:45 +0200 Subject: [PATCH] Python asset parser --- backend/Cargo.lock | 1 + backend/parsers/windmill-parser-py/Cargo.toml | 3 +- .../windmill-parser-py/src/asset_parser.rs | 81 +++++++++++++++++++ backend/parsers/windmill-parser-py/src/lib.rs | 3 + .../parsers/windmill-parser-wasm/src/lib.rs | 11 +++ .../src/lib/components/ScriptEditor.svelte | 2 +- .../flows/content/FlowModuleComponent.svelte | 10 ++- .../lib/components/graph/FlowGraphV2.svelte | 20 ++--- frontend/src/lib/infer.ts | 6 +- frontend/src/lib/svelte5Utils.svelte.ts | 5 +- 10 files changed, 124 insertions(+), 18 deletions(-) create mode 100644 backend/parsers/windmill-parser-py/src/asset_parser.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0518ddfb45..71795239a7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15064,6 +15064,7 @@ version = "1.502.2" dependencies = [ "anyhow", "itertools 0.14.0", + "rustpython-ast", "rustpython-parser", "serde_json", "windmill-parser", diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 9111534406..0c50e0f8d8 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -13,4 +13,5 @@ windmill-parser.workspace = true rustpython-parser.workspace = true itertools.workspace = true serde_json.workspace = true -anyhow.workspace = true \ No newline at end of file +anyhow.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs new file mode 100644 index 0000000000..e6285214bf --- /dev/null +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -0,0 +1,81 @@ +use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; +use rustpython_parser::{ast::Suite, Parse}; +use windmill_parser::asset_parser::{ + merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, +}; +use AssetUsageAccessType::*; + +pub fn parse_assets<'a>( + input: &'a str, + paths_storage: &'a mut Vec, +) -> anyhow::Result>> { + let ast = Suite::parse(&input, "main.py") + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + + let mut assets_finder = AssetsFinder { assets: vec![], paths_storage }; + ast.into_iter() + .for_each(|stmt| assets_finder.visit_stmt(stmt)); + for (asset, path) in assets_finder + .assets + .iter_mut() + .zip(assets_finder.paths_storage.iter_mut()) + { + asset.path = path; + } + Ok(merge_assets(assets_finder.assets)) +} + +struct AssetsFinder<'a> { + assets: Vec>, + // We have to store paths separately because of lifetime concerns + paths_storage: &'a mut Vec, +} +impl<'a> Visitor for AssetsFinder<'a> { + fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { + match self.visit_expr_call_inner(&node) { + Ok(_) => {} + Err(_) => self.generic_visit_expr_call(node), + } + } +} + +impl<'a> AssetsFinder<'a> { + fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + let ident: String = node + .func + .as_name_expr() + .and_then(|o| o.id.parse().ok()) + .or_else(|| { + node.func + .as_attribute_expr() + .and_then(|attr| attr.value.as_name_expr().and_then(|o| o.id.parse().ok())) + }) + .ok_or(())?; + + let (kind, access_type) = match ident.as_str() { + "get_resource" => (AssetKind::Resource, None), + "load_s3_file" => (AssetKind::S3Object, Some(R)), + "write_s3_file" => (AssetKind::S3Object, Some(W)), + _ => return Err(()), + }; + + if node.args.len() < 1 { + return Err(()); + } + + match &node.args[0] { + Expr::Constant(ExprConstant { value: Constant::Str(value), .. }) => { + if let Some((k, path)) = parse_asset_syntax(value.as_str()) { + if k != kind { + return Err(()); + } + self.paths_storage.push(path.to_string()); + self.assets + .push(ParseAssetsResult { kind, path: "", access_type }); + } + } + _ => return Err(()), + }; + Ok(()) + } +} diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 22d2ea6516..9b5736a18d 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -20,6 +20,9 @@ use rustpython_parser::{ Parse, }; +pub mod asset_parser; +pub use asset_parser::parse_assets; + const FUNCTION_CALL: &str = ""; fn filter_non_main(code: &str, main_name: &str) -> String { diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 98c9c594f0..67d6ddf792 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -189,4 +189,15 @@ pub fn parse_assets_ts(code: &str) -> String { } } +#[cfg(feature = "py-parser")] +#[wasm_bindgen] +pub fn parse_assets_py(code: &str) -> String { + let mut paths_storage = vec![]; + if let Ok(r) = windmill_parser_py::parse_assets(code, &mut paths_storage) { + return serde_json::to_string(&r).unwrap(); + } else { + return "Invalid".to_string(); + } +} + // for related places search: ADD_NEW_LANG diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 1212145df7..a089b4d1c5 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -135,7 +135,7 @@ dispatch('change', { code, schema }) }) - let assets = usePromise(() => inferAssets(lang, code)) + let assets = usePromise(() => inferAssets(lang, code), { clearValueOnRefresh: false }) $effect(() => { untrack(() => assets.refresh()), [lang, code] }) diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 23fa393314..3cac461f00 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -299,10 +299,12 @@ } }) - let assets = usePromise(async () => - flowModule.value.type === 'rawscript' - ? await inferAssets(flowModule.value.language, flowModule.value.content) - : undefined + let assets = usePromise( + async () => + flowModule.value.type === 'rawscript' + ? await inferAssets(flowModule.value.language, flowModule.value.content) + : undefined, + { clearValueOnRefresh: false } ) $effect(() => { if (flowModule.value.type !== 'rawscript') return diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index f913cf0f78..1bee63a221 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -749,15 +749,17 @@ key={[v.content, v.asset_alternative_access_types]} runFirstEffect onChange={() => - inferAssets(v.language, v.content).then((assets) => { - for (const override of v.asset_alternative_access_types ?? []) { - assets = assets.map((asset) => { - if (assetEq(asset, override)) return { ...asset, access_type: override.access_type } - return asset - }) - } - if (assetsMap && !deepEqual(assetsMap[mod.id], assets)) assetsMap[mod.id] = assets - })} + inferAssets(v.language, v.content) + .then((assets) => { + for (const override of v.asset_alternative_access_types ?? []) { + assets = assets.map((asset) => { + if (assetEq(asset, override)) return { ...asset, access_type: override.access_type } + return asset + }) + } + if (assetsMap && !deepEqual(assetsMap[mod.id], assets)) assetsMap[mod.id] = assets + }) + .catch((e) => {})} /> {/if} {/each} diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index ee998b3e04..68bbc848fc 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -19,7 +19,7 @@ import initRegexParsers, { parse_powershell, parse_assets_sql } from 'windmill-parser-wasm-regex' -import initPythonParser, { parse_python } from 'windmill-parser-wasm-py' +import initPythonParser, { parse_assets_py, parse_python } from 'windmill-parser-wasm-py' import initGoParser, { parse_go } from 'windmill-parser-wasm-go' import initPhpParser, { parse_php } from 'windmill-parser-wasm-php' import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust' @@ -92,6 +92,10 @@ export async function inferAssets( await initWasmTs() return JSON.parse(parse_assets_ts(code)) } + if (language === 'python3') { + await initWasmPython() + return JSON.parse(parse_assets_py(code)) + } return [] } diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index 9a486e66a1..956ee42541 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -28,11 +28,12 @@ export type UsePromiseResult = ( export type UsePromiseOptions = { loadInit?: boolean + clearValueOnRefresh?: boolean } export function usePromise( createPromise: () => Promise, - { loadInit = true }: UsePromiseOptions = {} + { loadInit = true, clearValueOnRefresh = true }: UsePromiseOptions = {} ): UsePromiseResult { const ret: any = $state({ status: 'loading', @@ -41,7 +42,7 @@ export function usePromise( let promise = createPromise() ret.__promise = promise ret.status = 'loading' - ret.value = undefined + if (clearValueOnRefresh) ret.value = undefined ret.error = undefined promise