feat(python): cache dependency resolution

backend/migrations/20230408203759_cache_pip_resolution.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE pip_resolution_cache;

backend/migrations/20230408203759_cache_pip_resolution.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+CREATE TABLE pip_resolution_cache(
+    hash VARCHAR(255) PRIMARY KEY,
+    expiration TIMESTAMP NOT NULL,
+    lockfile TEXT NOT NULL
+);

backend/sqlx-data.json

@@ -1319,6 +1319,24 @@
     },
     "query": "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE name = $1 AND workspace_id = $2"
   },
+  "399a8337a2488fa2ce3da2ef3281a34f8f96ee0d833c2fe3c22a0aa43e306f09": {
+    "describe": {
+      "columns": [
+        {
+          "name": "hash",
+          "ordinal": 0,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": []
+      }
+    },
+    "query": "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash"
+  },
   "39bae7dff750565183ac3893b51a8846d7db2f130a6795b54ef94acc232dec8b": {
     "describe": {
       "columns": [
@@ -3008,6 +3026,26 @@
     },
     "query": "SELECT workspace_id, name, display_name, owners, extra_perms FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3"
   },
+  "866b26e56fd376368c759db6eee13f92373e308f584903121b4546ad51aef86e": {
+    "describe": {
+      "columns": [
+        {
+          "name": "lockfile",
+          "ordinal": 0,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1"
+  },
   "8876fa929ffb175cd976a2bca1195704aa9fe7215013ae29e49ef15cb201ba57": {
     "describe": {
       "columns": [],
@@ -3836,6 +3874,19 @@
     },
     "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n        app.extra_perms, app_version.value, \n        app_version.created_at, app_version.created_by from app, app_version \n        WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]"
   },
+  "9baf1f88ab6c1df642cf8f0f49805ccac3e48cce1f0e184fb9db78536a8ca7b0": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Text"
+        ]
+      }
+    },
+    "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2"
+  },
   "9db64c9ff790d8c833c1e831a87803c32103ce9de68cc9c08d6f56cc988d7e37": {
     "describe": {
       "columns": [

backend/windmill-common/src/utils.rs

@@ -8,6 +8,7 @@
 
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 use serde::Deserialize;
+use sha2::{Digest, Sha256};
 
 use crate::error::{Error, Result};
 
@@ -118,3 +119,9 @@ pub fn rd_string(len: usize) -> String {
         .map(char::from)
         .collect()
 }
+
+pub fn calculate_hash(s: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(s);
+    format!("{:x}", hasher.finalize())
+}

backend/windmill-worker/src/worker.rs

@@ -25,7 +25,7 @@ use windmill_common::{
     error::{self, to_anyhow, Error},
     flows::{FlowModuleValue, FlowValue},
     scripts::{ScriptHash, ScriptLang},
-    utils::rd_string,
+    utils::{rd_string, calculate_hash},
     variables, BASE_URL, users::SUPERADMIN_SECRET_EMAIL, IS_READY,
 };
 use windmill_queue::{canceled_job_to_result, get_queued_job, pull, JobKind, QueuedJob, CLOUD_HOSTED};
@@ -2281,6 +2281,14 @@ async fn pip_compile(
     logs.push_str(&format!("\nresolving dependencies..."));
     set_logs(logs, job_id, db).await;
     logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
+    let req_hash = calculate_hash(&requirements) ;
+    if let Some(cached) = sqlx::query_scalar!(
+        "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+        req_hash    
+    ).fetch_optional(db).await? {
+        logs.push_str(&format!("\nfound cached resolution"));
+        return Ok(cached);
+    }
     let file = "requirements.in";
     let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
         let deps = pip_local_dependencies.clone();
@@ -2316,12 +2324,18 @@ async fn pip_compile(
     let mut file = File::open(path_lock).await?;
     let mut req_content = "".to_string();
     file.read_to_string(&mut req_content).await?;
-    Ok(req_content
+    let lockfile = req_content
         .lines()
         .filter(|x| !x.trim_start().starts_with('#'))
         .map(|x| x.to_string())
         .collect::<Vec<String>>()
-        .join("\n"))
+        .join("\n");
+    sqlx::query!(
+        "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() - ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
+        req_hash,
+        lockfile
+    ).fetch_optional(db).await?;
+    Ok(lockfile)
 }
 
 async fn install_go_dependencies(