From 3618e26326bc89bda1de080e609402c3deca5e0b Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 12 Dec 2024 00:32:03 +0100 Subject: [PATCH] Add no_uv_usage stats (#4883) * fix no_uv not affecting deploy Before this fix no_uv, no_uv_compile and no_uv_install were not affecting Dependency jobs These jobs are only affected if used USE_PIP_COMPILE or USE_PIP_INSTALL env variables To make it more consistant, no_uv should also affect dep jobs. Also make ansible use uv by default * Make it build * Add no_uv_usage stats * Provide build-env for Samael through shell.nix * Run update_sqlx.sh * Update ee-repo-ref * Update sqlx cache * Update ee-repo-ref * Update ee repo ref * Update sqlx cache * Update ee-repo-ref.txt --- ...259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json | 32 +++++++ ...b6ad60469d5a4949729676abdefb3ab2b1a7f.json | 14 +++ ...64a77334a0aad0cbf219250e4e811b1d16052.json | 14 +++ backend/ee-repo-ref.txt | 2 +- .../windmill-worker/src/ansible_executor.rs | 4 +- .../windmill-worker/src/python_executor.rs | 4 +- .../windmill-worker/src/worker_lockfiles.rs | 91 ++++++++++++++----- shell.nix | 34 +++++++ 8 files changed, 165 insertions(+), 30 deletions(-) create mode 100644 backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json create mode 100644 backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json create mode 100644 backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json diff --git a/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json b/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json new file mode 100644 index 0000000000..134cb58d17 --- /dev/null +++ b/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'python' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_py'\n\n UNION ALL\n \n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'ansible' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_ansible'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_deploy", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "deploys_count", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "language", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0" +} diff --git a/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json b/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json new file mode 100644 index 0000000000..d1fad59f5a --- /dev/null +++ b/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f" +} diff --git a/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json b/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json new file mode 100644 index 0000000000..89ef99521b --- /dev/null +++ b/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 32341c9e08..9d6ca729e0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a07bc62582c809457f1c945d6cda145770b94d04 \ No newline at end of file +89a221f8b6e0dc431c668ed804066c97b428f6ef diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index dcc288f108..c2d917ad86 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -116,8 +116,8 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), - true, - true, + false, + false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 586dd9f05e..3c31fb1913 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -51,11 +51,11 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); - static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); /// Use pip install - static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") + pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 959757d117..0f66376d8f 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -13,7 +13,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations}; use windmill_common::{ apps::AppScriptId, error::{self, to_anyhow}, @@ -28,7 +28,9 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; +use crate::python_executor::{ + create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL, +}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_bun_lockfile, @@ -1574,6 +1576,8 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + no_uv_compile: bool, + no_uv_install: bool, ) -> std::result::Result { create_dependencies_dir(job_dir).await; let req: std::result::Result = uv_pip_compile( @@ -1586,7 +1590,7 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - false, + no_uv_compile, false, ) .await; @@ -1603,7 +1607,7 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - false, + no_uv_install, false, ) .await; @@ -1654,28 +1658,23 @@ async fn capture_dependency_job( .join("\n") }; - python_dep( - reqs, - job_id, - mem_peak, - canceled_by, - job_dir, - db, - worker_name, - w_id, - worker_dir, - &mut Some(occupancy_metrics), - ) - .await - } - ScriptLang::Ansible => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for ansible".to_string(), - )); + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = + PythonAnnotations::parse(job_raw_code); + + if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_py', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e); + } } - let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; - let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); python_dep( reqs, @@ -1688,6 +1687,48 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + no_uv_compile | no_uv, + no_uv_install | no_uv, + ) + .await + } + ScriptLang::Ansible => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for ansible".to_string(), + )); + } + let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; + let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); + + if *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_ansible', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e); + }; + } + + python_dep( + reqs, + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + false, + false, ) .await } diff --git a/shell.nix b/shell.nix index ad49c3f898..149e8bdae8 100644 --- a/shell.nix +++ b/shell.nix @@ -7,6 +7,8 @@ let rust_overlay = import (builtins.fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"); pkgs = import { overlays = [ rust_overlay ]; }; + lib = pkgs.lib; + stdenv = pkgs.stdenv; # TODO: Pin version? # rustVersion = "latest"; rustVersion = "2024-09-30"; @@ -58,4 +60,36 @@ in pkgs.mkShell { # Useful for development RUST_LOG = "debug"; + # ---- Samael ---- + # https://github.com/njaremko/samael/blob/master/flake.nix#L104-L119 + # Otherwise samael crate will fail to build + # Need to tell bindgen where to find libclang + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + + # Set C flags for Rust's bindgen program. Unlike ordinary C + # compilation, bindgen does not invoke $CC directly. Instead it + # uses LLVM's libclang. To make sure all necessary flags are + # included we need to look in a few places. + # See https://web.archive.org/web/20220523141208/https://hoverbear.org/blog/rust-bindgen-in-nix/ + BINDGEN_EXTRA_CLANG_ARGS = + "${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${ + builtins.readFile "${stdenv.cc}/nix-support/libc-cflags" + } ${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"} ${ + builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags" + } -idirafter ${pkgs.libiconv}/include ${ + lib.optionalString stdenv.cc.isClang + "-idirafter ${stdenv.cc.cc}/lib/clang/${ + lib.getVersion stdenv.cc.cc + }/include" + } ${ + lib.optionalString stdenv.cc.isGNU + "-isystem ${stdenv.cc.cc}/include/c++/${ + lib.getVersion stdenv.cc.cc + } -isystem ${stdenv.cc.cc}/include/c++/${ + lib.getVersion stdenv.cc.cc + }/${stdenv.hostPlatform.config} -idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${ + lib.getVersion stdenv.cc.cc + }/include" + }"; + # ---- Samael ---- }