kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lock lockfiles for roles and collections

Browse Source
This commit is contained in:
wendrul
2025-04-23 21:31:26 +02:00
parent c6f1bde64f
commit 7e5d725ac9
3 changed files with 112 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
backend/parsers/windmill-parser-yaml/src/lib.rs
View File

@@ -1,3 +1,5 @@
use std::collections::HashMap;
use anyhow::anyhow;
use serde_json::json;
use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
@@ -220,7 +222,7 @@ pub struct GitRepo {
#[derive(Debug, Clone)]
pub struct AnsibleRequirements {
pub python_reqs: Vec<String>,
pub collections: Option<String>,
pub roles_and_collections: Option<String>,
pub file_resources: Vec<FileResource>,
pub inventories: Vec<AnsibleInventory>,
pub vars: Vec<(String, String)>,
@@ -236,7 +238,7 @@ impl Default for AnsibleRequirements {
fn default() -> Self {
Self {
python_reqs: vec![],
collections: None,
roles_and_collections: None,
file_resources: vec![],
inventories: vec![],
vars: vec![],
@@ -325,7 +327,7 @@ pub fn parse_ansible_reqs(
let mut out_str = String::new();
let mut emitter = YamlEmitter::new(&mut out_str);
emitter.dump(galaxy_requirements)?;
ret.collections = Some(out_str);
ret.roles_and_collections = Some(out_str);
}
if let Some(Yaml::Array(py_reqs)) =
deps.get(&Yaml::String("python".to_string()))
@@ -632,3 +634,78 @@ fn yaml_to_json(yaml: &Yaml) -> serde_json::Value {
_ => serde_json::Value::Null,
}
}
fn update_versions(
section: &str,
yaml: &mut Yaml,
versions: &HashMap<String, String>,
) -> anyhow::Result<String> {
let mut logs = String::new();
let Yaml::Hash(ref mut m) = yaml else {
return Err(anyhow!("{section} dependency should be a map"));
};
if let Some(Yaml::Array(elements)) = m.get_mut(&Yaml::String(section.to_string())) {
for el in elements {
let Yaml::Hash(ref mut h) = el else {
return Err(anyhow!("{section} dependency element should be a map"));
};
if let Some(name) = h
.get(&Yaml::String("name".to_string()))
.and_then(|n| n.as_str())
{
if let Some(version) = versions.get(name) {
h.insert(
Yaml::String("version".to_string()),
Yaml::String(version.to_string()),
);
} else {
logs.push_str(&format!("WARNING: {section} dependency `{name}` has no locked version, using the latest or system installed version.\n"));
}
} else {
return Err(anyhow!(
"{section} dependency element: missing or invalid `name` field"
));
}
}
}
Ok(logs)
}
pub fn add_versions_to_requirements_yaml(
input: &str,
role_versions: &HashMap<String, String>,
collection_versions: &HashMap<String, String>,
) -> anyhow::Result<(String,String)> {
let mut docs =
YamlLoader::load_from_str(input).map_err(|e| anyhow!("YAML parse error: {}", e))?;
let doc = &mut docs[0];
let mut logs = String::new();
logs.push_str(
&update_versions("roles", doc, role_versions)
.map_err(|e| anyhow!("Error updating role versions: {e}"))?,
);
logs.push_str(
&update_versions("collections", doc, collection_versions)
.map_err(|e| anyhow!("Error updating role versions: {e}"))?,
);
if !logs.is_empty() {
logs.push_str("WARNING: You might want to try adding manual versions for these, otherwise there could be breaking changes on deployed scripts\n");
}
let mut out_str = String::new();
{
let mut emitter = YamlEmitter::new(&mut out_str);
emitter
.dump(doc)
.map_err(|e| anyhow!("YAML emit error: {}", e))?;
}
Ok((out_str, logs))
}

22
backend/windmill-worker/src/ansible_executor.rs
View File

@@ -511,8 +511,10 @@ pub async fn install_galaxy_collections(
pub struct AnsibleDependencyLocks {
pub python_lockfile: String,
pub git_repos: HashMap<String, String>, // URL to full commit hash
pub collection_versions: HashMap<String, String>, //
pub role_versions: HashMap<String, String>,
pub collections_and_roles: String,
pub collections_and_roles_logs: String,
// pub collection_versions: HashMap<String, String>, //
// pub role_versions: HashMap<String, String>,
}
pub async fn get_collection_locks(
@@ -574,7 +576,6 @@ pub async fn get_role_locks(job_dir: &str) -> anyhow::Result<(HashMap<String, St
if !output.status.success() {
let stderr = String::from_utf8(output.stderr)?;
// return Err(anyhow!("Error getting ansible role versions: {stderr}"));
logs.push_str(&format!("Error getting ansible role versions: {stderr}"));
return Ok((ret, logs));
}
@@ -931,9 +932,20 @@ pub async fn handle_ansible_job(
.await;
}
if let Some(collections) = r.collections.as_ref() {
if let Some(collections) = r.roles_and_collections.as_ref() {
let empty = String::new();
let (lockfile, logs) =
req_lockfiles
.as_ref()
.map(|r| (&r.collections_and_roles, &r.collections_and_roles_logs))
.unwrap_or((collections, &empty));
if !logs.is_empty() {
append_logs(&job.id, &job.workspace_id, logs, conn).await;
}
install_galaxy_collections(
collections,
lockfile,
job_dir,
&job.id,
worker_name,

24
backend/windmill-worker/src/worker_lockfiles.rs
View File

@@ -1823,6 +1823,8 @@ async fn ansible_dep(
token: &str,
base_internal_url: &str,
) -> std::result::Result<String, Error> {
use windmill_parser_yaml::add_versions_to_requirements_yaml;
use crate::{
ansible_executor::{
create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks,
@@ -1876,7 +1878,7 @@ async fn ansible_dep(
create_ansible_cfg(Some(&reqs), job_dir, false)?;
if let Some(collections) = reqs.collections.as_ref() {
if let Some(collections) = reqs.roles_and_collections.as_ref() {
install_galaxy_collections(
collections,
job_dir,
@@ -1891,24 +1893,28 @@ async fn ansible_dep(
)
.await?;
let (collection_versions, logs) = get_collection_locks(job_dir).await?;
append_logs(job_id, w_id, logs, conn).await;
let (collection_versions, logs1) = get_collection_locks(job_dir).await?;
let (role_versions, logs) = get_role_locks(job_dir).await?;
append_logs(job_id, w_id, logs, conn).await;
let (role_versions, logs2) = get_role_locks(job_dir).await?;
let (reqs_yaml, logs3) = add_versions_to_requirements_yaml(&collections, &role_versions, &collection_versions)?;
let logs = format!("\n{logs1}\n{logs2}\n{logs3}\n");
append_logs(job_id, w_id, &logs, conn).await;
ansible_lockfile = AnsibleDependencyLocks {
python_lockfile,
git_repos,
collection_versions,
role_versions,
collections_and_roles: reqs_yaml,
collections_and_roles_logs: logs,
};
} else {
ansible_lockfile = AnsibleDependencyLocks {
python_lockfile,
git_repos,
collection_versions: HashMap::new(),
role_versions: HashMap::new(),
collections_and_roles: String::new(),
collections_and_roles_logs: String::new(),
};
}