From dfcfcda7b929c1e2620aaed774d820b8bede5b15 Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 17 Feb 2025 23:49:03 +0300 Subject: [PATCH] perf: fetch only new inputs on refresh of inputs history --- backend/windmill-api/openapi.yaml | 4 +++ backend/windmill-api/src/inputs.rs | 14 ++++++-- .../src/lib/components/HistoricList.svelte | 35 ++++++++++++++++--- .../src/lib/components/InfiniteList.svelte | 8 ++--- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 87af4c763d..4fffcfd696 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10159,6 +10159,10 @@ paths: in: query schema: type: boolean + - name: since + in: query + schema: + type: string responses: "200": description: Input history for completed jobs diff --git a/backend/windmill-api/src/inputs.rs b/backend/windmill-api/src/inputs.rs index ec200b4464..2345f6b3b6 100644 --- a/backend/windmill-api/src/inputs.rs +++ b/backend/windmill-api/src/inputs.rs @@ -18,11 +18,12 @@ use serde_json::Value; use sqlx::{types::Uuid, FromRow}; use std::{ fmt::{Display, Formatter}, + str::FromStr, vec, }; use windmill_common::{ db::UserDB, - error::JsonResult, + error::{self, JsonResult}, jobs::JobKind, scripts::to_i64, utils::{not_found_if_none, paginate, Pagination}, @@ -118,6 +119,7 @@ pub struct CompletedJobMini { #[derive(Deserialize)] struct GetInputHistory { include_preview: Option, + since: Option, } async fn get_input_history( @@ -135,7 +137,8 @@ async fn get_input_history( let sql = &format!( "select id, v2_job.created_at, created_by, 'null'::jsonb as args, status = 'success' as success from v2_job JOIN v2_job_completed USING (id) \ where {} = $1 and kind = any($2) and v2_job.workspace_id = $3 AND v2_job_completed.status != 'skipped' \ - order by v2_job.created_at desc limit $4 offset $5", + AND v2_job_completed.started_at >= $4 \ + order by v2_job.created_at desc limit $5 offset $6", r.runnable_type.column_name() ); @@ -156,9 +159,16 @@ async fn get_input_history( kind => vec![kind], }; + let since: DateTime = if let Some(date) = g.since { + DateTime::from_str(date.as_str()).map_err(|e| error::Error::BadRequest(format!("{e}")))? + } else { + DateTime::default() + }; + let rows = query .bind(job_kinds) .bind(&w_id) + .bind(since) .bind(per_page as i32) .bind(offset as i32) .fetch_all(&mut *tx) diff --git a/frontend/src/lib/components/HistoricList.svelte b/frontend/src/lib/components/HistoricList.svelte index c3118d3900..b5fa44a539 100644 --- a/frontend/src/lib/components/HistoricList.svelte +++ b/frontend/src/lib/components/HistoricList.svelte @@ -10,7 +10,9 @@ export let selected: string | undefined = undefined let infiniteList: InfiniteList | undefined = undefined - let loadInputsPageFn: ((page: number, perPage: number) => Promise) | undefined = undefined + let loadInputsPageFn: + | ((page: number, perPage: number, discovery: boolean) => Promise) + | undefined = undefined export function refresh() { if (infiniteList) { @@ -18,6 +20,10 @@ } } let cachedArgs: Record = {} + let items: any[] = [] + let potentialItems: any[] = [] + let perPageBind = 0 + let lastChecked: string | undefined = undefined let interval: NodeJS.Timeout | undefined = undefined function initLoadInputs() { @@ -25,15 +31,19 @@ interval = setInterval(() => { refresh() }, 10000) - loadInputsPageFn = async (page: number, perPage: number) => { - const inputs = await InputService.getInputHistory({ + loadInputsPageFn = async (page: number, perPage: number, discovery: boolean) => { + const request = InputService.getInputHistory({ workspace: $workspaceStore!, runnableId, runnableType, page, perPage, - includePreview: true + includePreview: true, + // If it is discovery, then we would like to fetch all values + since: !discovery ? lastChecked : undefined }) + if (!discovery) lastChecked = new Date().toJSON() + const inputs = await request const inputsWithPayload = await Promise.all( inputs.map(async (input) => { @@ -58,7 +68,22 @@ } }) ) - return inputsWithPayload + if (!discovery) { + // Add new items to beginning + items.unshift(...inputsWithPayload) + // We need to know when to apply potential items, + // it only happens when InfiniteList decides to expand list + // + // We cannot apply potential items right after fetch, + // because that would trigger expansion in list + // and old items will be loading by 10 every reload + if (perPageBind != perPage) items.push(...potentialItems), (perPageBind = perPage) + return items + } else { + // Save discovered items to buffer and apply later + potentialItems = inputsWithPayload + return potentialItems + } } infiniteList?.setLoader(loadInputsPageFn) } diff --git a/frontend/src/lib/components/InfiniteList.svelte b/frontend/src/lib/components/InfiniteList.svelte index e8caaefdb7..9bf425017c 100644 --- a/frontend/src/lib/components/InfiniteList.svelte +++ b/frontend/src/lib/components/InfiniteList.svelte @@ -18,7 +18,7 @@ let hasAlreadyFailed = false let hovered: any | undefined = undefined let initLoad = false - let loadInputs: ((page: number, perPage: number) => Promise) | undefined = undefined + let loadInputs: ((page: number, perPage: number, discovery: boolean) => Promise) | undefined = undefined let deleteItemFn: ((id: any) => Promise) | undefined = undefined export async function loadData(loadOption: 'refresh' | 'forceRefresh' | 'loadMore' = 'loadMore') { @@ -33,7 +33,7 @@ } try { - const newItems = await loadInputs(1, page * perPage) + const newItems = await loadInputs(1, page * perPage, false) if ( loadOption === 'refresh' && @@ -63,7 +63,7 @@ page = Math.ceil(items.length / perPage) hasMore = items.length === perPage * page if (hasMore) { - const potentialNewItems = await loadInputs(page + 1, perPage) + const potentialNewItems = await loadInputs(page + 1, perPage, true) hasMore = potentialNewItems.length > 0 } initLoad = true @@ -96,7 +96,7 @@ } } - export async function setLoader(loader: (page: number, perPage: number) => Promise) { + export async function setLoader(loader: (page: number, perPage: number, discovery: boolean) => Promise) { loadInputs = loader loadData('forceRefresh') }