kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

further memory reduction

Browse Source
This commit is contained in:
Ruben Fiszel
2024-05-20 11:38:11 +02:00
parent 46c440f41a
commit cd8b14a2fa
3 changed files with 31 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/windmill-api/src/apps.rs
View File

@@ -105,12 +105,12 @@ pub struct AppWithLastVersion {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: serde_json::Value,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: serde_json::Value,
pub extra_perms: Option<serde_json::Value>,
}
#[derive(Serialize, Deserialize, FromRow)]
@@ -118,7 +118,7 @@ pub struct AppWithLastVersionAndDraft {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: serde_json::Value,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
@@ -437,7 +437,7 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
null as extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
@@ -448,7 +448,7 @@ async fn get_public_app_by_secret(
let app = not_found_if_none(app_o, "App", id.to_string())?;
let policy = serde_json::from_value::<Policy>(app.policy.clone()).map_err(to_anyhow)?;
let policy = serde_json::from_str::<Policy>(app.policy.0.get()).map_err(to_anyhow)?;
if !matches!(policy.execution_mode, ExecutionMode::Anonymous) {
return Err(Error::NotAuthorized(
@@ -650,7 +650,7 @@ async fn list_hub_apps(Extension(db): Extension<DB>) -> impl IntoResponse {
pub async fn get_hub_app_by_id(
Path(id): Path<i32>,
Extension(db): Extension<DB>,
) -> JsonResult<serde_json::Value> {
) -> JsonResult<Box<serde_json::value::RawValue>> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id),

2
backend/windmill-api/src/flows.rs
View File

@@ -199,7 +199,7 @@ async fn list_paths(
pub async fn get_hub_flow_by_id(
Path(id): Path<i32>,
Extension(db): Extension<DB>,
) -> JsonResult<serde_json::Value> {
) -> JsonResult<Box<serde_json::value::RawValue>> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id),

36
backend/windmill-api/src/jobs.rs
View File

@@ -3785,7 +3785,7 @@ pub struct JobUpdate {
pub new_logs: Option<String>,
pub log_offset: Option<i32>,
pub mem_peak: Option<i32>,
pub flow_status: Option<serde_json::Value>,
pub flow_status: Option<Box<serde_json::value::RawValue>>,
}
async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result<Response> {
@@ -3838,22 +3838,30 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R
)));
}
#[derive(Deserialize, sqlx::FromRow)]
pub struct JobUpdateRow {
pub running: bool,
pub logs: Option<String>,
pub mem_peak: Option<i32>,
pub flow_status: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
pub log_offset: Option<i32>,
}
async fn get_job_update(
Extension(db): Extension<DB>,
Path((w_id, job_id)): Path<(String, Uuid)>,
Query(JobUpdateQuery { running, log_offset }): Query<JobUpdateQuery>,
) -> error::JsonResult<JobUpdate> {
let record = sqlx::query!(
let record = sqlx::query_as::<_, JobUpdateRow>(
"SELECT running, substr(concat(coalesce(queue.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status,
job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset
FROM queue
LEFT JOIN job_logs ON job_logs.job_id = queue.id
WHERE queue.workspace_id = $2 AND queue.id = $3",
log_offset,
&w_id,
&job_id
)
.bind(log_offset)
.bind(&w_id)
.bind(&job_id)
.fetch_optional(&db)
.await?;
@@ -3868,20 +3876,22 @@ async fn get_job_update(
completed: None,
new_logs: record.logs,
mem_peak: record.mem_peak,
flow_status: record.flow_status,
flow_status: record
.flow_status
.map(|x: sqlx::types::Json<Box<RawValue>>| x.0),
}))
} else {
let record = sqlx::query!(
"SELECT substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
let record = sqlx::query_as::<_, JobUpdateRow>(
"SELECT false as running, substr(concat(coalesce(completed_job.logs, ''), job_logs.logs), greatest($1 - job_logs.log_offset, 0)) as logs, mem_peak,
CASE WHEN is_flow_step is true then NULL else flow_status END as flow_status,
job_logs.log_offset + char_length(job_logs.logs) + 1 as log_offset
FROM completed_job
LEFT JOIN job_logs ON job_logs.job_id = completed_job.id
WHERE completed_job.workspace_id = $2 AND id = $3",
log_offset,
&w_id,
&job_id
)
.bind(log_offset)
.bind(&w_id)
.bind(&job_id)
.fetch_optional(&db)
.await?;
if let Some(record) = record {
@@ -3891,7 +3901,9 @@ async fn get_job_update(
log_offset: record.log_offset,
new_logs: record.logs,
mem_peak: record.mem_peak,
flow_status: record.flow_status,
flow_status: record
.flow_status
.map(|x: sqlx::types::Json<Box<RawValue>>| x.0),
}))
} else {
Err(error::Error::NotFound(format!("Job not found: {}", job_id)))