kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: invalidate result cache on flow or script change

Browse Source
This commit is contained in:
Ruben Fiszel
2023-10-31 12:49:31 +01:00
parent c98ebf92e5
commit cf9669c18d
6 changed files with 122 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/Cargo.lock generated
View File

@@ -8358,6 +8358,7 @@ dependencies = [
"futures",
"gcp_auth",
"git-version",
"hex",
"itertools 0.11.0",
"jsonwebtoken",
"lazy_static",

1
backend/windmill-worker/Cargo.toml
View File

@@ -73,6 +73,7 @@ urlencoding.workspace = true
nix.workspace = true
bytes.workspace = true
reqwest.workspace = true
hex.workspace = true
[build-dependencies]
deno_fetch.workspace = true

6
backend/windmill-worker/src/common.rs
View File

@@ -780,7 +780,7 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
}
}
pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> i64 {
pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> String {
if let Some(vs) = v {
let mut dh = DefaultHasher::new();
let hm = &vs.0;
@@ -788,9 +788,9 @@ pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>)
k.hash(&mut dh);
hm.get(k).unwrap().get().hash(&mut dh);
}
dh.finish() as i64
hex::encode(dh.finish().to_be_bytes())
} else {
0
"empty_args".to_string()
}
}

27
backend/windmill-worker/src/worker.rs
View File

@@ -17,7 +17,8 @@ use reqwest::Response;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sqlx::{types::Json, Pool, Postgres};
use std::{
collections::HashMap,
collections::{hash_map::DefaultHasher, HashMap},
hash::Hash,
sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Arc,
@@ -2096,6 +2097,23 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
};
let cached_res_path = if job.cache_ttl.is_some() {
let version_hash = if let Some(h) = job.script_hash {
format!("script_{}", h.to_string())
} else if let Some(rc) = job.raw_code.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
rc.hash(&mut s);
format!("inline_{}", hex::encode(s.finish().to_be_bytes()))
} else if let Some(rc) = job.raw_flow.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
serde_json::to_string(&rc.0)
.unwrap_or_default()
.hash(&mut s);
format!("flow_{}", hex::encode(s.finish().to_be_bytes()))
} else {
"none".to_string()
};
let args_hash = hash_args(&job.args);
if job.is_flow_step {
let flow_path = sqlx::query_scalar!(
@@ -2107,10 +2125,11 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
.map_err(|e| Error::InternalErr(format!("fetching step flow status: {e}")))?
.ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?;
let step = step.unwrap_or(-1);
Some(format!("{flow_path}/cache/{step}/{args_hash}"))
Some(format!(
"{flow_path}/cache/{version_hash}/{step}/{args_hash}"
))
} else if let Some(script_path) = &job.script_path {
let is_flow = if job.is_flow() { "flow/" } else { "" };
Some(format!("{script_path}/{is_flow}cache/{args_hash}"))
Some(format!("{script_path}/cache/{version_hash}/{args_hash}"))
} else {
None
}

14
backend/windmill-worker/src/worker_flow.rs
View File

@@ -6,7 +6,9 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::hash::Hash;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
@@ -615,7 +617,17 @@ pub async fn update_flow_status_after_job_completion_internal<
let cached_res_path = {
let args_hash = hash_args(&flow_job.args);
let flow_path = flow_job.script_path();
format!("{flow_path}/flow/cache/{args_hash}")
let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
serde_json::to_string(&rc.0)
.unwrap_or_default()
.hash(&mut s);
format!("flow_{}", hex::encode(s.finish().to_be_bytes()))
} else {
"flow_unknown".to_string()
};
format!("{flow_path}/cache/{version_hash}/{args_hash}")
};
save_in_cache(db, &flow_job, cached_res_path, &nresult).await;

130
frontend/src/routes/(root)/(logged)/resources/+page.svelte
View File

@@ -39,6 +39,7 @@
faPen,
faPlus,
faRefresh,
faRotateRight,
faSave,
faShare,
faTrash
@@ -111,7 +112,9 @@
typeFilter == undefined
? preFilteredItemsOwners?.filter((x) =>
tab === 'workspace'
? x.resource_type !== 'app_theme' && x.resource_type !== 'state'
? x.resource_type !== 'app_theme' &&
x.resource_type !== 'state' &&
x.resource_type !== 'cache'
: tab === 'states'
? x.resource_type === 'state'
: tab === 'cache'
@@ -124,7 +127,9 @@
(x) =>
x.resource_type === typeFilter &&
(tab === 'workspace'
? x.resource_type !== 'app_theme' && x.resource_type !== 'state'
? x.resource_type !== 'app_theme' &&
x.resource_type !== 'state' &&
x.resource_type !== 'cache'
: true)
)
@@ -440,53 +445,78 @@
</Button>
</div>
</PageHeader>
<Tabs bind:selected={tab}>
<Tab size="md" value="workspace">
<div class="flex gap-2 items-center my-1">
<Building size={18} />
Workspace
</div>
</Tab>
<Tab size="md" value="types">
<div class="flex gap-2 items-center my-1">
Resource Types
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/resources_and_types"
>
Every resources have Resource Types attached to them which contains its schema and make it
easy in scripts and flows to accept only resources of a specific resource type
</Tooltip>
</div>
</Tab>
<Tab size="md" value="states">
<div class="flex gap-2 items-center my-1">
States
<Tooltip>
States are actually resources (but excluded from the Workspace tab for clarity). States
are used by scripts to keep data persistent between runs of the same script by the same
trigger (schedule or user)
</Tooltip>
</div>
</Tab>
<Tab size="md" value="cache">
<div class="flex gap-2 items-center my-1">
Cache
<Tooltip>
Cached results are actually resources (but excluded from the Workspace tab for clarity).
Cache are used by flows's step to cache result to avoid recomputing unnecessarily
</Tooltip>
</div>
</Tab>
<Tab size="md" value="theme">
<div class="flex gap-2 items-center my-1">
Theme
<Tooltip>
Theme are actually resources (but excluded from the Workspace tab for clarity). Theme are
used by the apps to customize their look and feel.
</Tooltip>
</div>
</Tab>
</Tabs>
<div class="flex justify-between">
<Tabs class="w-full" bind:selected={tab}>
<Tab size="md" value="workspace">
<div class="flex gap-2 items-center my-1">
<Building size={18} />
Workspace
</div>
</Tab>
<Tab size="md" value="types">
<div class="flex gap-2 items-center my-1">
Resource Types
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/resources_and_types"
>
Every resources have Resource Types attached to them which contains its schema and make
it easy in scripts and flows to accept only resources of a specific resource type
</Tooltip>
</div>
</Tab>
<Tab size="md" value="states">
<div class="flex gap-2 items-center my-1">
States
<Tooltip>
States are actually resources (but excluded from the Workspace tab for clarity). States
are used by scripts to keep data persistent between runs of the same script by the same
trigger (schedule or user)
</Tooltip>
</div>
</Tab>
<Tab size="md" value="cache">
<div class="flex gap-2 items-center my-1">
Cache
<Tooltip>
Cached results are actually resources (but excluded from the Workspace tab for clarity).
Cache are used by flows's step to cache result to avoid recomputing unnecessarily
</Tooltip>
</div>
</Tab>
<Tab size="md" value="theme">
<div class="flex gap-2 items-center my-1">
Theme
<Tooltip>
Theme are actually resources (but excluded from the Workspace tab for clarity). Theme
are used by the apps to customize their look and feel.
</Tooltip>
</div>
</Tab>
</Tabs>
<div class="flex">
<Button
variant="border"
color="light"
on:click={async () => {
loading = {
resources: true,
types: true
}
await loadResources()
await loadResourceTypes()
loading = {
resources: false,
types: false
}
}}
><Icon
scale={0.8}
data={faRotateRight}
class={loading.resources || loading.types ? 'animate-spin' : ''}
/></Button
>
</div>
</div>
{#if tab == 'workspace' || tab == 'states' || tab == 'cache' || tab == 'theme'}
<div class="pt-2">
<input placeholder="Search Resource" bind:value={filter} class="input mt-1" />
@@ -499,6 +529,8 @@
filters={types}
resourceType
/>
{:else}
<div class="h-4" />
{/if}
<div class="overflow-x-auto pb-40">