From 7e5d725ac96c97d9fc50c2f54fcb36abf4cff6ca Mon Sep 17 00:00:00 2001 From: wendrul Date: Wed, 23 Apr 2025 21:31:26 +0200 Subject: [PATCH] Lock lockfiles for roles and collections --- .../parsers/windmill-parser-yaml/src/lib.rs | 83 ++++++++++++++++++- .../windmill-worker/src/ansible_executor.rs | 22 +++-- .../windmill-worker/src/worker_lockfiles.rs | 24 ++++-- 3 files changed, 112 insertions(+), 17 deletions(-) diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index e36c30df7c..f2c8e14e21 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use anyhow::anyhow; use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; @@ -220,7 +222,7 @@ pub struct GitRepo { #[derive(Debug, Clone)] pub struct AnsibleRequirements { pub python_reqs: Vec, - pub collections: Option, + pub roles_and_collections: Option, pub file_resources: Vec, pub inventories: Vec, pub vars: Vec<(String, String)>, @@ -236,7 +238,7 @@ impl Default for AnsibleRequirements { fn default() -> Self { Self { python_reqs: vec![], - collections: None, + roles_and_collections: None, file_resources: vec![], inventories: vec![], vars: vec![], @@ -325,7 +327,7 @@ pub fn parse_ansible_reqs( let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); emitter.dump(galaxy_requirements)?; - ret.collections = Some(out_str); + ret.roles_and_collections = Some(out_str); } if let Some(Yaml::Array(py_reqs)) = deps.get(&Yaml::String("python".to_string())) @@ -632,3 +634,78 @@ fn yaml_to_json(yaml: &Yaml) -> serde_json::Value { _ => serde_json::Value::Null, } } + +fn update_versions( + section: &str, + yaml: &mut Yaml, + versions: &HashMap, +) -> anyhow::Result { + let mut logs = String::new(); + + let Yaml::Hash(ref mut m) = yaml else { + return Err(anyhow!("{section} dependency should be a map")); + }; + + if let Some(Yaml::Array(elements)) = m.get_mut(&Yaml::String(section.to_string())) { + for el in elements { + let Yaml::Hash(ref mut h) = el else { + return Err(anyhow!("{section} dependency element should be a map")); + }; + + if let Some(name) = h + .get(&Yaml::String("name".to_string())) + .and_then(|n| n.as_str()) + { + if let Some(version) = versions.get(name) { + h.insert( + Yaml::String("version".to_string()), + Yaml::String(version.to_string()), + ); + } else { + logs.push_str(&format!("WARNING: {section} dependency `{name}` has no locked version, using the latest or system installed version.\n")); + } + } else { + return Err(anyhow!( + "{section} dependency element: missing or invalid `name` field" + )); + } + } + } + + Ok(logs) +} + +pub fn add_versions_to_requirements_yaml( + input: &str, + role_versions: &HashMap, + collection_versions: &HashMap, +) -> anyhow::Result<(String,String)> { + let mut docs = + YamlLoader::load_from_str(input).map_err(|e| anyhow!("YAML parse error: {}", e))?; + let doc = &mut docs[0]; + + let mut logs = String::new(); + + logs.push_str( + &update_versions("roles", doc, role_versions) + .map_err(|e| anyhow!("Error updating role versions: {e}"))?, + ); + logs.push_str( + &update_versions("collections", doc, collection_versions) + .map_err(|e| anyhow!("Error updating role versions: {e}"))?, + ); + + if !logs.is_empty() { + logs.push_str("WARNING: You might want to try adding manual versions for these, otherwise there could be breaking changes on deployed scripts\n"); + } + + let mut out_str = String::new(); + { + let mut emitter = YamlEmitter::new(&mut out_str); + emitter + .dump(doc) + .map_err(|e| anyhow!("YAML emit error: {}", e))?; + } + + Ok((out_str, logs)) +} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 8eaa4b5316..2a3e3c3282 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -511,8 +511,10 @@ pub async fn install_galaxy_collections( pub struct AnsibleDependencyLocks { pub python_lockfile: String, pub git_repos: HashMap, // URL to full commit hash - pub collection_versions: HashMap, // - pub role_versions: HashMap, + pub collections_and_roles: String, + pub collections_and_roles_logs: String, + // pub collection_versions: HashMap, // + // pub role_versions: HashMap, } pub async fn get_collection_locks( @@ -574,7 +576,6 @@ pub async fn get_role_locks(job_dir: &str) -> anyhow::Result<(HashMap std::result::Result { + use windmill_parser_yaml::add_versions_to_requirements_yaml; + use crate::{ ansible_executor::{ create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, @@ -1876,7 +1878,7 @@ async fn ansible_dep( create_ansible_cfg(Some(&reqs), job_dir, false)?; - if let Some(collections) = reqs.collections.as_ref() { + if let Some(collections) = reqs.roles_and_collections.as_ref() { install_galaxy_collections( collections, job_dir, @@ -1891,24 +1893,28 @@ async fn ansible_dep( ) .await?; - let (collection_versions, logs) = get_collection_locks(job_dir).await?; - append_logs(job_id, w_id, logs, conn).await; + let (collection_versions, logs1) = get_collection_locks(job_dir).await?; - let (role_versions, logs) = get_role_locks(job_dir).await?; - append_logs(job_id, w_id, logs, conn).await; + let (role_versions, logs2) = get_role_locks(job_dir).await?; + + let (reqs_yaml, logs3) = add_versions_to_requirements_yaml(&collections, &role_versions, &collection_versions)?; + + let logs = format!("\n{logs1}\n{logs2}\n{logs3}\n"); + + append_logs(job_id, w_id, &logs, conn).await; ansible_lockfile = AnsibleDependencyLocks { python_lockfile, git_repos, - collection_versions, - role_versions, + collections_and_roles: reqs_yaml, + collections_and_roles_logs: logs, }; } else { ansible_lockfile = AnsibleDependencyLocks { python_lockfile, git_repos, - collection_versions: HashMap::new(), - role_versions: HashMap::new(), + collections_and_roles: String::new(), + collections_and_roles_logs: String::new(), }; }