Compare commits

...

4 Commits

Author SHA1 Message Date
Ruben Fiszel
e33241da5a chore: add SKIP_MIN_VERSION_CHECK env var and database URL docs
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-28 17:24:53 +00:00
Ruben Fiszel
4920e8afaf fix: exclude preview jobs from fast_filter (only script/flow/singlestepflow)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-27 07:02:38 +00:00
Ruben Fiszel
3fb977a71a fix: add fast_filter to all v2_job_completed INSERT paths
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-27 06:56:58 +00:00
Ruben Fiszel
86581d73d1 feat: add fast_filter column to v2_job_completed for runs page optimization
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-27 06:45:40 +00:00
18 changed files with 275 additions and 66 deletions

View File

@@ -51,10 +51,15 @@ After making code changes, you MUST run the appropriate checks and fix all error
`backend/summarized_schema.txt` provides a compact overview of all tables, columns, types, ENUMs, and foreign keys. Use it to quickly understand the data model and relationships. Note: this file is a simplified summary — it omits indexes, constraints details, and other metadata.
For exact table definitions (indexes, constraints, column defaults, etc.), query the database directly:
For exact table definitions (indexes, constraints, column defaults, etc.), query the database directly.
**IMPORTANT**: The database name varies per worktree/branch. Always read `.env.local` first to get the correct `DATABASE_URL`. The default `windmill` database is only used on the main branch — worktrees use branch-specific database names (e.g., `windmill_add_fast_filter`). Using the wrong database means your queries, inserts, and migrations will target stale or empty data.
```bash
psql postgres://postgres:changeme@localhost:5432/windmill
# Always check .env.local first:
grep DATABASE_URL .env.local
# Then use the correct database URL, e.g.:
psql "postgres://postgres:changeme@127.0.0.1:5432/windmill_add_fast_filter"
```
Useful psql commands:

View File

@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "WITH batch AS (\n SELECT c.id FROM v2_job_completed c\n JOIN v2_job j ON c.id = j.id\n WHERE j.parent_job IS NULL AND c.fast_filter IS NULL AND c.status != 'skipped'\n AND j.kind IN ('script', 'flow', 'singlestepflow')\n LIMIT 50000\n )\n UPDATE v2_job_completed SET fast_filter =\n CASE WHEN v2_job_completed.status = 'success' THEN 1::smallint ELSE 2::smallint END\n FROM batch WHERE v2_job_completed.id = batch.id",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "308a85372588477b92cacac56017c9f8112529e676dfb4a8851567ec9842957c"
}

View File

@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Jsonb",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c"
}

View File

@@ -1,25 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , duration_ms\n , result\n , canceled_by\n , canceled_reason\n , flow_status\n , status\n , worker\n )\n SELECT q.workspace_id\n , q.id\n , 0\n , $4\n , $1\n , 'cancel all'\n , (SELECT flow_status FROM v2_job_status WHERE id = q.id)\n , 'canceled'::job_status\n , worker\n FROM v2_job_queue q\n JOIN v2_job USING (id)\n WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule'\n FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Varchar",
"UuidArray",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "77f13b7c1e7e488c6268a9ff6def647e47394bd0a61d44fddb1b2040d05b7f17"
}

View File

@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n , fast_filter\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker,\n $11::smallint\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3, fast_filter = EXCLUDED.fast_filter RETURNING duration_ms AS \"duration_ms!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Jsonb",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Int2"
]
},
"nullable": [
false
]
},
"hash": "96edefb17b81ab9f7c9f41dd3d5a3663c700583c265fe42edecf8d15cf33b0d0"
}

View File

@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO windmill_migrations (name) VALUES ('fast_filter_indexed')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "ad6d4ffdf39ff238f200323134871ab6cc94fe422eaaff8b48e934136cf4cd16"
}

View File

@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , duration_ms\n , result\n , canceled_by\n , canceled_reason\n , flow_status\n , status\n , worker\n , fast_filter\n )\n SELECT q.workspace_id\n , q.id\n , 0\n , $4\n , $1\n , 'cancel all'\n , (SELECT flow_status FROM v2_job_status WHERE id = q.id)\n , 'canceled'::job_status\n , worker\n , CASE WHEN kind IN ('script', 'flow', 'singlestepflow') THEN 2::smallint ELSE NULL END\n FROM v2_job_queue q\n JOIN v2_job USING (id)\n WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule'\n FOR UPDATE SKIP LOCKED\n ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Varchar",
"UuidArray",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "e8315dace910c9454a79ef15b9ac6057675c0c697cbd9aa4220eb318fc9515eb"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO v2_job_completed (\n id, workspace_id, started_at, completed_at, duration_ms, result, deleted,\n canceled_by, canceled_reason, flow_status, memory_peak, status, worker,\n workflow_as_code_status, result_columns, retries, extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ",
"query": "\n INSERT INTO v2_job_completed (\n id, workspace_id, started_at, completed_at, duration_ms, result, deleted,\n canceled_by, canceled_reason, flow_status, memory_peak, status, worker,\n workflow_as_code_status, result_columns, retries, extras, fast_filter\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18\n )\n ON CONFLICT (id) DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
@@ -33,10 +33,11 @@
"Jsonb",
"TextArray",
"UuidArray",
"Jsonb"
"Jsonb",
"Int2"
]
},
"nullable": []
},
"hash": "141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e"
"hash": "ee47ef28bf714c90dba387b49f92c96bcad6046385da58aa753fb9c89a8cbb63"
}

View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fast_filter_indexed')",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "f614ef418f7124c789ec17f97f0cf7bed89553063512434119523e89abda3c41"
}

View File

@@ -0,0 +1 @@
ALTER TABLE v2_job_completed DROP COLUMN IF EXISTS fast_filter;

View File

@@ -0,0 +1 @@
ALTER TABLE v2_job_completed ADD COLUMN IF NOT EXISTS fast_filter SMALLINT;

View File

@@ -99,6 +99,7 @@ pub async fn cancel_jobs(
, flow_status
, status
, worker
, fast_filter
)
SELECT q.workspace_id
, q.id
@@ -109,6 +110,7 @@ pub async fn cancel_jobs(
, (SELECT flow_status FROM v2_job_status WHERE id = q.id)
, 'canceled'::job_status
, worker
, CASE WHEN kind IN ('script', 'flow', 'singlestepflow') THEN 2::smallint ELSE NULL END
FROM v2_job_queue q
JOIN v2_job USING (id)
WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule'

View File

@@ -358,14 +358,27 @@ pub async fn import_completed_jobs(
.execute(&mut *tx)
.await?;
let fast_filter: Option<i16> = if job.parent_job.is_some()
|| job.status == JobStatus::Skipped
|| !matches!(
job.kind,
JobKind::Script | JobKind::Flow | JobKind::SingleStepFlow
) {
None
} else if job.status == JobStatus::Success {
Some(1)
} else {
Some(2)
};
sqlx::query!(
r#"
INSERT INTO v2_job_completed (
id, workspace_id, started_at, completed_at, duration_ms, result, deleted,
canceled_by, canceled_reason, flow_status, memory_peak, status, worker,
workflow_as_code_status, result_columns, retries, extras
workflow_as_code_status, result_columns, retries, extras, fast_filter
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
)
ON CONFLICT (id) DO NOTHING
"#,
@@ -385,7 +398,8 @@ pub async fn import_completed_jobs(
job.workflow_as_code_status as _,
job.result_columns as _,
job.retries as _,
job.extras as _
job.extras as _,
fast_filter
)
.execute(&mut *tx)
.await?;

View File

@@ -8,9 +8,12 @@
//! Query builders for filtering job lists (queue and completed).
use std::sync::atomic::Ordering;
use sql_builder::prelude::*;
use sql_builder::SqlBuilder;
use windmill_common::utils::{escape_ilike_pattern, paginate_without_limits, Pagination};
use windmill_common::FAST_FILTER_INDEXED;
use crate::types::{ListCompletedQuery, ListQueueQuery};
@@ -491,6 +494,14 @@ pub fn filter_list_completed_query(
if let Some(fs) = &lq.has_null_parent {
if *fs {
sqlb.and_where_is_null("parent_job");
if FAST_FILTER_INDEXED.load(Ordering::Relaxed) {
if lq.success == Some(false) {
sqlb.and_where("fast_filter = 2");
} else if lq.success.is_none() && lq.is_skipped == Some(false) {
sqlb.and_where("fast_filter IS NOT NULL");
}
}
}
}
if let Some(jk) = &lq.job_kinds {

View File

@@ -6,8 +6,12 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::sync::atomic::Ordering;
use sqlx::Postgres;
use windmill_common::error::Error;
use windmill_common::min_version::MIN_VERSION_IS_AT_LEAST_1_647;
use windmill_common::FAST_FILTER_INDEXED;
use crate::db::{CustomMigrator, DB};
use sqlx::migrate::Migrate;
@@ -18,6 +22,10 @@ pub async fn custom_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result
tracing::error!("Could not apply flow versioning fix migration: {err:#}");
}
if let Err(err) = fast_filter_migration(db).await {
tracing::error!("Could not start fast_filter migration: {err:#}");
}
Ok(())
}
@@ -67,3 +75,106 @@ async fn fix_flow_versioning_migration(
}
Ok(())
}
async fn fast_filter_migration(db: &DB) -> Result<(), Error> {
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fast_filter_indexed')",
)
.fetch_one(db)
.await?
.unwrap_or(false);
if has_done_migration {
FAST_FILTER_INDEXED.store(true, Ordering::Relaxed);
return Ok(());
}
let db = db.clone();
tokio::spawn(async move {
if let Err(e) = fast_filter_migration_inner(&db).await {
tracing::error!("fast_filter background migration failed: {e:#}");
}
});
Ok(())
}
async fn fast_filter_migration_inner(db: &DB) -> Result<(), Error> {
// Wait until all workers are at least version 1.647 so new jobs write fast_filter
let skip_version_check = std::env::var("SKIP_MIN_VERSION_CHECK")
.map(|v| v == "1" || v == "true")
.unwrap_or(false);
if !skip_version_check {
loop {
if MIN_VERSION_IS_AT_LEAST_1_647.met().await {
break;
}
tracing::info!("fast_filter migration: waiting for all workers to be >= 1.647");
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
}
} else {
tracing::info!(
"fast_filter migration: skipping version check (SKIP_MIN_VERSION_CHECK=true)"
);
}
// Backfill existing rows in batches (skip skipped jobs — they stay NULL)
loop {
let rows_updated = sqlx::query_scalar!(
"WITH batch AS (
SELECT c.id FROM v2_job_completed c
JOIN v2_job j ON c.id = j.id
WHERE j.parent_job IS NULL AND c.fast_filter IS NULL AND c.status != 'skipped'
AND j.kind IN ('script', 'flow', 'singlestepflow')
LIMIT 50000
)
UPDATE v2_job_completed SET fast_filter =
CASE WHEN v2_job_completed.status = 'success' THEN 1::smallint ELSE 2::smallint END
FROM batch WHERE v2_job_completed.id = batch.id"
)
.execute(db)
.await
.map_err(|e| Error::internal_err(format!("fast_filter backfill failed: {e:#}")))?
.rows_affected();
tracing::info!("fast_filter migration: backfilled {rows_updated} rows");
if rows_updated == 0 {
break;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
// Create partial indexes concurrently (must be outside a transaction)
tracing::info!("fast_filter migration: creating indexes concurrently");
sqlx::query(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_v2_job_completed_fast_filter_not_null
ON v2_job_completed (workspace_id, completed_at DESC) WHERE fast_filter IS NOT NULL",
)
.execute(db)
.await
.map_err(|e| Error::internal_err(format!("fast_filter index creation failed: {e:#}")))?;
sqlx::query(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_v2_job_completed_fast_filter_failure
ON v2_job_completed (workspace_id, completed_at DESC) WHERE fast_filter = 2",
)
.execute(db)
.await
.map_err(|e| Error::internal_err(format!("fast_filter index creation failed: {e:#}")))?;
tracing::info!("fast_filter migration: indexes created, marking migration as done");
sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('fast_filter_indexed')")
.execute(db)
.await
.map_err(|e| {
Error::internal_err(format!("fast_filter migration flag insert failed: {e:#}"))
})?;
FAST_FILTER_INDEXED.store(true, Ordering::Relaxed);
tracing::info!("fast_filter migration: complete");
Ok(())
}

View File

@@ -196,6 +196,7 @@ lazy_static::lazy_static! {
pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
pub static ref FAST_FILTER_INDEXED: AtomicBool = AtomicBool::new(false);
pub static ref HUB_BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new(DEFAULT_HUB_BASE_URL.to_string()));

View File

@@ -5,6 +5,7 @@ use tokio::sync::RwLock;
// ============ Feature Definitions ============
pub const MIN_VERSION_IS_AT_LEAST_1_647: VC = vc(1, 647, 0, "Fast filter column");
pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing");
pub const MIN_VERSION_SUPPORTS_DEBOUNCING_V2: VC = vc(1, 597, 0, "Debouncing V2");
pub const MIN_VERSION_IS_AT_LEAST_1_595: VC = vc(1, 595, 0, "Flow status separate table");

View File

@@ -923,6 +923,19 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
return value;
}
let fast_filter: Option<i16> = if completed_job.parent_job.is_some()
|| skipped
|| !matches!(
completed_job.kind,
JobKind::Script | JobKind::Flow | JobKind::SingleStepFlow
) {
None
} else if canceled_by.is_some() || !success {
Some(2)
} else {
Some(1)
};
let duration = sqlx::query_scalar!(
"INSERT INTO v2_job_completed AS cj
( workspace_id
@@ -938,6 +951,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
, memory_peak
, status
, worker
, fast_filter
)
SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,
flow_status, workflow_as_code_status,
@@ -945,9 +959,10 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
WHEN $7::BOOL THEN 'skipped'::job_status
WHEN $2::BOOL THEN 'success'::job_status
ELSE 'failure'::job_status END AS status,
q.worker
q.worker,
$11::smallint
FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1
ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"",
ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3, fast_filter = EXCLUDED.fast_filter RETURNING duration_ms AS \"duration_ms!\"",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result as Json<&T>,
@@ -958,6 +973,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
/* $8 */ if mem_peak > 0 { Some(mem_peak) } else { None },
/* $9 */ duration,
/* $10 */ result_columns as Option<&Vec<String>>,
/* $11 */ fast_filter,
)
.fetch_optional(&mut *tx)
.warn_after_seconds(10)