diff --git a/backend/Cargo.lock b/backend/Cargo.lock index fe2be04c11..27becc8e83 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8358,6 +8358,7 @@ dependencies = [ "futures", "gcp_auth", "git-version", + "hex", "itertools 0.11.0", "jsonwebtoken", "lazy_static", diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index d54158d056..8eafe92b46 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -73,6 +73,7 @@ urlencoding.workspace = true nix.workspace = true bytes.workspace = true reqwest.workspace = true +hex.workspace = true [build-dependencies] deno_fetch.workspace = true diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1c7d1e555c..b494fe3f4f 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -780,7 +780,7 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { } } -pub fn hash_args(v: &Option>>>) -> i64 { +pub fn hash_args(v: &Option>>>) -> String { if let Some(vs) = v { let mut dh = DefaultHasher::new(); let hm = &vs.0; @@ -788,9 +788,9 @@ pub fn hash_args(v: &Option>>>) k.hash(&mut dh); hm.get(k).unwrap().get().hash(&mut dh); } - dh.finish() as i64 + hex::encode(dh.finish().to_be_bytes()) } else { - 0 + "empty_args".to_string() } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e85d3d3d26..7fb6939e39 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -17,7 +17,8 @@ use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::{types::Json, Pool, Postgres}; use std::{ - collections::HashMap, + collections::{hash_map::DefaultHasher, HashMap}, + hash::Hash, sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, @@ -2096,6 +2097,23 @@ async fn handle_queued_job( }; let cached_res_path = if job.cache_ttl.is_some() { + let version_hash = if let Some(h) = job.script_hash { + format!("script_{}", h.to_string()) + } else if let Some(rc) = job.raw_code.as_ref() { + use std::hash::Hasher; + let mut s = DefaultHasher::new(); + rc.hash(&mut s); + format!("inline_{}", hex::encode(s.finish().to_be_bytes())) + } else if let Some(rc) = job.raw_flow.as_ref() { + use std::hash::Hasher; + let mut s = DefaultHasher::new(); + serde_json::to_string(&rc.0) + .unwrap_or_default() + .hash(&mut s); + format!("flow_{}", hex::encode(s.finish().to_be_bytes())) + } else { + "none".to_string() + }; let args_hash = hash_args(&job.args); if job.is_flow_step { let flow_path = sqlx::query_scalar!( @@ -2107,10 +2125,11 @@ async fn handle_queued_job( .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e}")))? .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?; let step = step.unwrap_or(-1); - Some(format!("{flow_path}/cache/{step}/{args_hash}")) + Some(format!( + "{flow_path}/cache/{version_hash}/{step}/{args_hash}" + )) } else if let Some(script_path) = &job.script_path { - let is_flow = if job.is_flow() { "flow/" } else { "" }; - Some(format!("{script_path}/{is_flow}cache/{args_hash}")) + Some(format!("{script_path}/cache/{version_hash}/{args_hash}")) } else { None } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index e43848c3dc..36c4e7cc23 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -6,7 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ +use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; +use std::hash::Hash; use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::Duration; @@ -615,7 +617,17 @@ pub async fn update_flow_status_after_job_completion_internal< let cached_res_path = { let args_hash = hash_args(&flow_job.args); let flow_path = flow_job.script_path(); - format!("{flow_path}/flow/cache/{args_hash}") + let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() { + use std::hash::Hasher; + let mut s = DefaultHasher::new(); + serde_json::to_string(&rc.0) + .unwrap_or_default() + .hash(&mut s); + format!("flow_{}", hex::encode(s.finish().to_be_bytes())) + } else { + "flow_unknown".to_string() + }; + format!("{flow_path}/cache/{version_hash}/{args_hash}") }; save_in_cache(db, &flow_job, cached_res_path, &nresult).await; diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 92ae98485c..4b7169d8a4 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -39,6 +39,7 @@ faPen, faPlus, faRefresh, + faRotateRight, faSave, faShare, faTrash @@ -111,7 +112,9 @@ typeFilter == undefined ? preFilteredItemsOwners?.filter((x) => tab === 'workspace' - ? x.resource_type !== 'app_theme' && x.resource_type !== 'state' + ? x.resource_type !== 'app_theme' && + x.resource_type !== 'state' && + x.resource_type !== 'cache' : tab === 'states' ? x.resource_type === 'state' : tab === 'cache' @@ -124,7 +127,9 @@ (x) => x.resource_type === typeFilter && (tab === 'workspace' - ? x.resource_type !== 'app_theme' && x.resource_type !== 'state' + ? x.resource_type !== 'app_theme' && + x.resource_type !== 'state' && + x.resource_type !== 'cache' : true) ) @@ -440,53 +445,78 @@ - - -
- - Workspace -
-
- -
- Resource Types - - Every resources have Resource Types attached to them which contains its schema and make it - easy in scripts and flows to accept only resources of a specific resource type - -
-
- -
- States - - States are actually resources (but excluded from the Workspace tab for clarity). States - are used by scripts to keep data persistent between runs of the same script by the same - trigger (schedule or user) - -
-
- -
- Cache - - Cached results are actually resources (but excluded from the Workspace tab for clarity). - Cache are used by flows's step to cache result to avoid recomputing unnecessarily - -
-
- -
- Theme - - Theme are actually resources (but excluded from the Workspace tab for clarity). Theme are - used by the apps to customize their look and feel. - -
-
-
+
+ + +
+ + Workspace +
+
+ +
+ Resource Types + + Every resources have Resource Types attached to them which contains its schema and make + it easy in scripts and flows to accept only resources of a specific resource type + +
+
+ +
+ States + + States are actually resources (but excluded from the Workspace tab for clarity). States + are used by scripts to keep data persistent between runs of the same script by the same + trigger (schedule or user) + +
+
+ +
+ Cache + + Cached results are actually resources (but excluded from the Workspace tab for clarity). + Cache are used by flows's step to cache result to avoid recomputing unnecessarily + +
+
+ +
+ Theme + + Theme are actually resources (but excluded from the Workspace tab for clarity). Theme + are used by the apps to customize their look and feel. + +
+
+
+
+ +
+
{#if tab == 'workspace' || tab == 'states' || tab == 'cache' || tab == 'theme'}
@@ -499,6 +529,8 @@ filters={types} resourceType /> + {:else} +
{/if}