Compare commits

...

12 Commits

Author SHA1 Message Date
pyranota
c4dfe6798c Merge branch 'workspace-dependencies-cache-rework' of github.com:windmill-labs/windmill into workspace-dependencies-cache-rework 2025-12-02 13:44:12 +01:00
pyranota
0b71ace5ad fix issue
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 13:44:01 +01:00
Pyra
15c9364d08 Merge branch 'main' into workspace-dependencies-cache-rework 2025-12-02 13:34:50 +01:00
pyranota
6dc53f385d fix ci
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 13:33:56 +01:00
pyranota
a2457dc9f2 nit
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 13:23:15 +01:00
pyranota
7dbf05865d spawn non blocking
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 13:14:07 +01:00
pyranota
d92c0e96ba Merge branch 'workspace-dependencies-cache-rework' of github.com:windmill-labs/windmill into workspace-dependencies-cache-rework 2025-12-02 12:40:53 +01:00
Pyra
4544b6be54 Merge branch 'main' into workspace-dependencies-cache-rework 2025-12-02 12:40:47 +01:00
pyranota
1613e6957d Merge branch 'workspace-dependencies-cache-rework' of github.com:windmill-labs/windmill into workspace-dependencies-cache-rework 2025-12-02 12:40:39 +01:00
pyranota
a976d963a8 nits
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 12:40:29 +01:00
windmill-internal-app[bot]
2179c78761 Update SQLx metadata 2025-12-02 11:40:17 +00:00
pyranota
2eeb568966 fix(workspace-dependencies): implement better caching
Signed-off-by: pyranota <pyra@duck.com>
2025-12-02 12:34:31 +01:00
7 changed files with 366 additions and 275 deletions

View File

@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false
]
},
"hash": "5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de"
}

View File

@@ -1,128 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, description, archived, workspace_id, created_at\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
},
{
"ordinal": 3,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "archived",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby"
]
}
}
}
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false
]
},
"hash": "f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc"
}

View File

@@ -64,7 +64,7 @@ mod workspace_dependencies {
name: Some("test".to_owned()),
description: None,
}
.create("", "", "", db)
.create(("".to_owned(), "".to_owned(), "".to_owned()), db.clone())
.await
.unwrap();
@@ -151,7 +151,7 @@ mod workspace_dependencies {
name: None, // No name = default workspace dependencies
description: None,
}
.create("", "", "", db)
.create(("".to_owned(), "".to_owned(), "".to_owned()), db.clone())
.await
.unwrap();

View File

@@ -13,9 +13,8 @@ use windmill_common::{
workspace_dependencies::WorkspaceDependencies,
DB,
};
use windmill_worker::{
scoped_dependency_map, trigger_dependents_to_recompute_dependencies,
workspace_dependencies::NewWorkspaceDependencies,
use windmill_worker::workspace_dependencies::{
trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies,
};
use crate::db::ApiAuthed;
@@ -43,10 +42,12 @@ async fn create(
format!(
"{}",
nwd.create(
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
&db
(
authed.email,
username_to_permissioned_as(&authed.username),
authed.username,
),
db
)
.await?
),
@@ -93,23 +94,20 @@ async fn archive(
let db = &db;
WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
trigger_dependents_to_recompute_dependencies_in_the_background(
params.name.is_none(),
w_id,
language,
(
authed.email,
username_to_permissioned_as(&authed.username),
authed.username,
),
WorkspaceDependencies::to_path(&params.name, language)?,
db.clone(),
)
.await
.await;
Ok(())
}
#[axum::debug_handler]
@@ -125,21 +123,18 @@ async fn delete(
let db = &db;
WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?;
trigger_dependents_to_recompute_dependencies(
&w_id,
scoped_dependency_map::ScopedDependencyMap::get_dependents(
WorkspaceDependencies::to_path(&params.name, language)?.as_str(),
&w_id,
db,
)
.await?,
None,
None,
&authed.email,
&authed.username,
&username_to_permissioned_as(&authed.username),
db,
vec![],
trigger_dependents_to_recompute_dependencies_in_the_background(
params.name.is_none(),
w_id,
language,
(
authed.email,
username_to_permissioned_as(&authed.username),
authed.username,
),
WorkspaceDependencies::to_path(&params.name, language)?,
db.clone(),
)
.await
.await;
Ok(())
}

View File

@@ -892,6 +892,86 @@ pub mod job {
}
}
pub mod workspace_dependencies {
use std::{
future::Future,
time::{Duration, Instant},
};
use crate::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies, DB};
make_static! {
/// Workspace Dependencies by id and workspace cache.
static ref WORKSPACE_DEPENDENCIES: { (i64, String) => WorkspaceDependencies } in "workspace_dependencies" <= 1000;
}
lazy_static::lazy_static! {
/// Cache for checking if default/unnamed workspace dependencies exist for a workspace and language.
/// Cache key: (workspace_id, language)
/// Cache value: (exists: bool, cached_at timestamp)
static ref DEFAULT_WD_EXISTS_CACHE: quick_cache::sync::Cache<(String, ScriptLang), (bool, Instant)> = quick_cache::sync::Cache::new(500);
}
/// Cache timeout for existence checks (10 seconds)
pub const EXISTS_CACHE_TIMEOUT: Duration = Duration::from_secs(10);
pub fn fetch_workspace_dependencies<'c>(
id: i64,
workspace_id: String,
db: &'c DB,
) -> impl Future<Output = error::Result<WorkspaceDependencies>> + 'c {
tracing::debug!(workspace_id = %workspace_id, id, "fetching workspace dependencies");
WORKSPACE_DEPENDENCIES.get_or_insert_async(
(id, workspace_id.clone()),
WorkspaceDependencies::get(id, workspace_id, db),
)
}
pub fn get_cached_is_unnamed_workspace_dependencies_exists<'c>(
language: ScriptLang,
workspace_id: String,
) -> Option<bool> {
let exists_key = (workspace_id.to_string(), language);
if let Some((exists, cached_at)) = DEFAULT_WD_EXISTS_CACHE.get(&exists_key) {
if cached_at.elapsed() < EXISTS_CACHE_TIMEOUT {
tracing::debug!(
workspace_id = %workspace_id,
?language,
exists,
"cache hit for unnamed workspace dependencies existence"
);
return Some(exists);
} else {
tracing::debug!(
workspace_id = %workspace_id,
?language,
"cache expired for unnamed workspace dependencies existence"
);
DEFAULT_WD_EXISTS_CACHE.remove(&exists_key);
}
} else {
tracing::debug!(
workspace_id = %workspace_id,
?language,
"cache miss for unnamed workspace dependencies existence"
);
}
None
}
pub fn set_cached_is_unnamed_workspace_dependencies_exists<'c>(
language: ScriptLang,
workspace_id: String,
exists: bool,
) {
tracing::debug!(
workspace_id = %workspace_id,
?language,
exists,
"setting cache for unnamed workspace dependencies existence"
);
let exists_key = (workspace_id.to_string(), language);
DEFAULT_WD_EXISTS_CACHE.insert(exists_key, (exists, Instant::now()));
}
}
const _: () = {
impl Import for RawFlow {
fn import(src: &impl Storage) -> error::Result<Self> {
@@ -1075,7 +1155,8 @@ const _: () = {
(ScriptHash, |x| format!("{:016x}", x.0)),
((u8, ScriptHash), |x| format!("{:02x}-{:016x}", x.0, x.1.0)),
(FlowNodeId, |x| format!("{:016x}", x.0)),
(AppScriptId, |x| format!("{:016x}", x.0))
(AppScriptId, |x| format!("{:016x}", x.0)),
((i64, String), |x| format!("{}-{}", x.1, x.0))
}
#[cfg(feature = "scoped_cache")]

View File

@@ -2,9 +2,17 @@ use itertools::Itertools;
use regex::Regex;
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
use std::time::{Duration, Instant};
use crate::{error, scripts::ScriptLang, utils::calculate_hash, worker::Connection};
use crate::{
cache::workspace_dependencies::{
fetch_workspace_dependencies, get_cached_is_unnamed_workspace_dependencies_exists,
set_cached_is_unnamed_workspace_dependencies_exists,
},
error,
scripts::ScriptLang,
utils::calculate_hash,
worker::Connection,
};
use phf::phf_set;
pub static BLACKLIST: phf::Set<&'static str> = phf_set! {
@@ -15,16 +23,8 @@ pub static BLACKLIST: phf::Set<&'static str> = phf_set! {
lazy_static::lazy_static! {
static ref WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: bool = std::env::var("WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES").is_ok();
/// Simple in-memory cache for workspace dependencies get_latest with 10-second timeout.
/// Cache key: (workspace_id, language, name)
/// Cache value: (Option<WorkspaceDependencies>, cached_at timestamp)
static ref WORKSPACE_DEPENDENCIES_CACHE: quick_cache::sync::Cache<(String, ScriptLang, Option<String>), (Option<WorkspaceDependencies>, Instant)> = quick_cache::sync::Cache::new(1000);
}
/// Cache timeout for workspace dependencies
const CACHE_TIMEOUT: Duration = Duration::from_secs(10);
/// Minimum Windmill version required for workspace dependencies feature
pub const MIN_VERSION_WORKSPACE_DEPENDENCIES: &str = "1.587.0";
@@ -174,6 +174,42 @@ impl WorkspaceDependencies {
.map_err(error::Error::from)
}
async fn get_latest_id<'c>(
name: Option<String>,
language: ScriptLang,
workspace_id: &str,
e: impl PgExecutor<'c>,
) -> error::Result<Option<i64>> {
tracing::debug!(
workspace_id = %workspace_id,
?language,
?name,
"fetching latest workspace dependencies id"
);
let result = sqlx::query_scalar!(
r#"
SELECT id FROM workspace_dependencies
WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3
LIMIT 1
"#,
name,
workspace_id,
language as ScriptLang
)
.fetch_optional(e)
.await
.map_err(error::Error::from)?;
tracing::debug!(
workspace_id = %workspace_id,
?language,
?name,
?result,
"fetched latest workspace dependencies id"
);
Ok(result)
}
/// Gets the latest version of workspace dependencies by name and language.
pub async fn get_latest(
name: Option<String>,
@@ -185,66 +221,84 @@ impl WorkspaceDependencies {
return Ok(None);
}
let cache_key = (workspace_id.to_string(), language, name.clone());
// Check if cached value is still valid
if let Some((cached_value, cached_at)) = WORKSPACE_DEPENDENCIES_CACHE.get(&cache_key) {
if cached_at.elapsed() < CACHE_TIMEOUT {
return Ok(cached_value);
}
// Expired, remove it
WORKSPACE_DEPENDENCIES_CACHE.remove(&cache_key);
if name.is_none()
&& get_cached_is_unnamed_workspace_dependencies_exists(
language,
workspace_id.to_owned(),
)
.map(|exists| exists == false)
.unwrap_or_default()
{
tracing::debug!(
workspace_id = %workspace_id,
?language,
"skipping unnamed workspace dependencies fetch - cached as non-existent"
);
return Ok(None);
}
// Fetch and cache
let fetch = Box::pin(async {
match &conn {
Connection::Sql(db) => sqlx::query_as!(
Self,
r#"
SELECT id, content, language AS "language: ScriptLang", name, description, archived, workspace_id, created_at
FROM workspace_dependencies
WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3
LIMIT 1
"#,
name,
workspace_id,
language as ScriptLang
)
.fetch_optional(db)
.await
.map_err(error::Error::from),
// Fetch from database or HTTP
let wd = match &conn {
Connection::Sql(db) => {
let Some(id) =
Self::get_latest_id(name.clone(), language, workspace_id, db).await?
else {
if name.is_none() {
set_cached_is_unnamed_workspace_dependencies_exists(
language,
workspace_id.to_owned(),
false,
);
}
Connection::Http(http_client) => http_client
.get::<Option<WorkspaceDependencies>>(&format!(
"/api/w/{workspace_id}/agent_workers/workspace_dependencies/get_latest/{}{}",
language.as_str(),
if let Some(ref name_val) = name {
format!("?name={name_val}")
} else {
"".to_owned()
}
))
.await
.map_err(error::Error::from),
tracing::debug!(
workspace_id = %workspace_id,
?language,
?name,
"no latest workspace dependencies found"
);
return Ok(None);
};
tracing::debug!(
workspace_id = %workspace_id,
?language,
?name,
id,
"fetching workspace dependencies by id from cache or db"
);
Some(fetch_workspace_dependencies(id, workspace_id.to_owned(), db).await?)
}
});
let (workspace_dependencies_o, ..) = WORKSPACE_DEPENDENCIES_CACHE
.get_or_insert_async(&cache_key, async {
Ok::<_, error::Error>((fetch.await?, Instant::now()))
})
.await?;
Connection::Http(http_client) => http_client
.get::<Option<WorkspaceDependencies>>(&format!(
"/api/w/{workspace_id}/agent_workers/workspace_dependencies/get_latest/{}{}",
language.as_str(),
if let Some(ref name_val) = name {
format!("?name={name_val}")
} else {
"".to_owned()
}
))
.await
.map_err(error::Error::from)?,
};
Ok(workspace_dependencies_o)
if name.is_none() {
set_cached_is_unnamed_workspace_dependencies_exists(
language,
workspace_id.to_owned(),
wd.is_some(),
);
}
Ok(wd)
}
/// Gets workspace dependencies by their unique ID.
pub async fn get<'c>(
id: i64,
workspace_id: &str,
workspace_id: String,
e: impl PgExecutor<'c>,
) -> error::Result<Option<Self>> {
) -> error::Result<Self> {
sqlx::query_as!(
Self,
r#"
@@ -254,9 +308,9 @@ impl WorkspaceDependencies {
LIMIT 1
"#,
id,
workspace_id
&workspace_id
)
.fetch_optional(e)
.fetch_one(e)
.await
.map_err(error::Error::from)
}

View File

@@ -1,5 +1,8 @@
use serde::{Deserialize, Serialize};
use windmill_common::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies};
use windmill_common::{
cache::workspace_dependencies::EXISTS_CACHE_TIMEOUT, error, scripts::ScriptLang,
workspace_dependencies::WorkspaceDependencies,
};
use crate::{
scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies,
@@ -25,10 +28,8 @@ impl NewWorkspaceDependencies {
/// and rebuilds the dependency map if this is the first unnamed dependency for the workspace.
pub async fn create<'c>(
self,
email: &str,
created_by: &str,
permissioned_as: &str,
db: &sqlx::Pool<sqlx::Postgres>,
metadata: (String, String, String),
db: sqlx::Pool<sqlx::Postgres>,
) -> error::Result<i64> {
// Check if all workers support workspace dependencies feature
windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies()
@@ -43,7 +44,7 @@ impl NewWorkspaceDependencies {
let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id);
let already_rebuilt =
windmill_common::global_settings::load_value_from_global_settings(
db,
&db,
&setting_name,
)
.await?
@@ -54,11 +55,11 @@ impl NewWorkspaceDependencies {
workspace_id = %self.workspace_id,
"Rebuilding workspace dependencies map for first unnamed workspace dependencies"
);
ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, db).await?;
ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, &db).await?;
// Mark as rebuilt by creating the setting
windmill_common::global_settings::set_value_in_global_settings(
db,
&db,
&setting_name,
serde_json::json!({}),
)
@@ -111,50 +112,83 @@ impl NewWorkspaceDependencies {
.await?;
tx.commit().await?;
// Make sure trigger dependents will have latest view.
// NOTE: Uncomment for tests
// #[cfg(test)]
// assert_eq!(
// sqlx::query_scalar!(
// "
// SELECT id FROM workspace_dependencies
// WHERE archived = false
// AND name IS NOT DISTINCT FROM $1
// AND workspace_id = $2
// AND language = $3
// ",
// self.name,
// self.workspace_id,
// self.language as ScriptLang,
// )
// .fetch_one(db) // Use db
// .await?,
// new_id
// );
// It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies
trigger_dependents_to_recompute_dependencies(
&self.workspace_id,
crate::scoped_dependency_map::ScopedDependencyMap::get_dependents(
path.as_str(),
&self.workspace_id,
db,
)
.await?,
None,
None,
email,
created_by,
permissioned_as,
trigger_dependents_to_recompute_dependencies_in_the_background(
prev_description.is_none() && self.name.is_none(),
self.workspace_id,
self.language,
metadata,
path,
db,
vec![],
)
.await?;
.await;
Ok(new_id)
}
}
pub async fn trigger_dependents_to_recompute_dependencies_in_the_background(
wait_for_cache_timeout: bool,
workspace_id: String,
language: ScriptLang,
(email, permissioned_as, created_by): (String, String, String),
path: String,
db: sqlx::Pool<sqlx::Postgres>,
) {
tokio::spawn(async move {
if wait_for_cache_timeout {
tracing::debug!(
workspace_id = %workspace_id,
language = ?language,
"waiting for cache timeout after creating first unnamed workspace dependencies"
);
// Wait for cache timeout.
// For context, workers have cache on whether the unnamed workspace dependencies exists or not.
// when we trigger dependents to recompoute dependencies we want to make sure all workers are having cache timed out.
// otherwise it would result into bug, when workers skip fetch of workspace dependencies because they think they don't exist.
tokio::time::sleep(EXISTS_CACHE_TIMEOUT).await;
}
// It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies
if let Err(e) = trigger_dependents_to_recompute_dependencies(
&workspace_id,
match crate::scoped_dependency_map::ScopedDependencyMap::get_dependents(
path.as_str(),
&workspace_id,
&db,
)
.await
{
Ok(importers) => importers,
Err(e) => {
tracing::error!(
workspace_id = %workspace_id,
path = %path,
error = %e,
"CRITICAL: failed to get dependents for workspace dependencies - dependent runnables are not being redeployed. Please contact the Windmill team"
);
return;
}
},
None,
None,
email.as_str(),
created_by.as_str(),
permissioned_as.as_str(),
&db,
vec![],
)
.await
{
tracing::error!(
workspace_id = %workspace_id,
path = %path,
error = %e,
"CRITICAL: failed to trigger dependents to recompute dependencies - dependent runnables are not being redeployed. Please contact the Windmill team"
);
}
});
}
// Type aliases for backward compatibility
pub type RawRequirements = WorkspaceDependencies;
pub type NewRawRequirements = NewWorkspaceDependencies;