From 130ddbbf85753f926f0019c39cd7370cf95cb2d1 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 8 Apr 2023 23:10:50 +0200 Subject: [PATCH] feat(python): cache dependency resolution --- ...230408203759_cache_pip_resolution.down.sql | 2 + ...20230408203759_cache_pip_resolution.up.sql | 6 +++ backend/sqlx-data.json | 51 +++++++++++++++++++ backend/windmill-common/src/utils.rs | 7 +++ backend/windmill-worker/src/worker.rs | 20 ++++++-- 5 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 backend/migrations/20230408203759_cache_pip_resolution.down.sql create mode 100644 backend/migrations/20230408203759_cache_pip_resolution.up.sql diff --git a/backend/migrations/20230408203759_cache_pip_resolution.down.sql b/backend/migrations/20230408203759_cache_pip_resolution.down.sql new file mode 100644 index 0000000000..4754b4ca4c --- /dev/null +++ b/backend/migrations/20230408203759_cache_pip_resolution.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE pip_resolution_cache; \ No newline at end of file diff --git a/backend/migrations/20230408203759_cache_pip_resolution.up.sql b/backend/migrations/20230408203759_cache_pip_resolution.up.sql new file mode 100644 index 0000000000..ecbfff3820 --- /dev/null +++ b/backend/migrations/20230408203759_cache_pip_resolution.up.sql @@ -0,0 +1,6 @@ +-- Add up migration script here +CREATE TABLE pip_resolution_cache( + hash VARCHAR(255) PRIMARY KEY, + expiration TIMESTAMP NOT NULL, + lockfile TEXT NOT NULL +); \ No newline at end of file diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index eb3b6c0e80..42ee3afd93 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -1319,6 +1319,24 @@ }, "query": "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE name = $1 AND workspace_id = $2" }, + "399a8337a2488fa2ce3da2ef3281a34f8f96ee0d833c2fe3c22a0aa43e306f09": { + "describe": { + "columns": [ + { + "name": "hash", + "ordinal": 0, + "type_info": "Varchar" + } + ], + "nullable": [ + false + ], + "parameters": { + "Left": [] + } + }, + "query": "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash" + }, "39bae7dff750565183ac3893b51a8846d7db2f130a6795b54ef94acc232dec8b": { "describe": { "columns": [ @@ -3008,6 +3026,26 @@ }, "query": "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3" }, + "866b26e56fd376368c759db6eee13f92373e308f584903121b4546ad51aef86e": { + "describe": { + "columns": [ + { + "name": "lockfile", + "ordinal": 0, + "type_info": "Text" + } + ], + "nullable": [ + false + ], + "parameters": { + "Left": [ + "Text" + ] + } + }, + "query": "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1" + }, "8876fa929ffb175cd976a2bca1195704aa9fe7215013ae29e49ef15cb201ba57": { "describe": { "columns": [], @@ -3836,6 +3874,19 @@ }, "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]" }, + "9baf1f88ab6c1df642cf8f0f49805ccac3e48cce1f0e184fb9db78536a8ca7b0": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + } + }, + "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2" + }, "9db64c9ff790d8c833c1e831a87803c32103ce9de68cc9c08d6f56cc988d7e37": { "describe": { "columns": [ diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 07253b0481..bf11ca909f 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -8,6 +8,7 @@ use rand::{distributions::Alphanumeric, thread_rng, Rng}; use serde::Deserialize; +use sha2::{Digest, Sha256}; use crate::error::{Error, Result}; @@ -118,3 +119,9 @@ pub fn rd_string(len: usize) -> String { .map(char::from) .collect() } + +pub fn calculate_hash(s: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(s); + format!("{:x}", hasher.finalize()) +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f622bf32f5..2ef5571384 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -25,7 +25,7 @@ use windmill_common::{ error::{self, to_anyhow, Error}, flows::{FlowModuleValue, FlowValue}, scripts::{ScriptHash, ScriptLang}, - utils::rd_string, + utils::{rd_string, calculate_hash}, variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL, IS_READY, }; use windmill_queue::{canceled_job_to_result, get_queued_job, pull, JobKind, QueuedJob, CLOUD_HOSTED}; @@ -2281,6 +2281,14 @@ async fn pip_compile( logs.push_str(&format!("\nresolving dependencies...")); set_logs(logs, job_id, db).await; logs.push_str(&format!("\ncontent of requirements:\n{}", requirements)); + let req_hash = calculate_hash(&requirements) ; + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ).fetch_optional(db).await? { + logs.push_str(&format!("\nfound cached resolution")); + return Ok(cached); + } let file = "requirements.in"; let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() { let deps = pip_local_dependencies.clone(); @@ -2316,12 +2324,18 @@ async fn pip_compile( let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; - Ok(req_content + let lockfile = req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) .map(|x| x.to_string()) .collect::>() - .join("\n")) + .join("\n"); + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + req_hash, + lockfile + ).fetch_optional(db).await?; + Ok(lockfile) } async fn install_go_dependencies(