feat: add mem peak info
This commit is contained in:
1
backend/migrations/20230204182500_add_mem_peak.down.sql
Normal file
1
backend/migrations/20230204182500_add_mem_peak.down.sql
Normal file
@@ -0,0 +1 @@
|
||||
-- Add down migration script here
|
||||
3
backend/migrations/20230204182500_add_mem_peak.up.sql
Normal file
3
backend/migrations/20230204182500_add_mem_peak.up.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE queue ADD COLUMN mem_peak INTEGER;
|
||||
ALTER TABLE completed_job ADD COLUMN mem_peak INTEGER;
|
||||
@@ -117,17 +117,25 @@
|
||||
},
|
||||
"query": "SELECT script_path FROM queue WHERE id = $1"
|
||||
},
|
||||
"062859f1d0e5cfba3115f4241115753b86a4ad239708851c998ff5620ebca5b8": {
|
||||
"0715955b7e98cc669a88eca6556cd46b7f9c07fcf32a24f85b69720b54e6e95f": {
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"columns": [
|
||||
{
|
||||
"name": "mem_peak",
|
||||
"ordinal": 0,
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
true
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "UPDATE queue SET last_ping = now() WHERE id = $1"
|
||||
"query": "SELECT mem_peak FROM queue WHERE id = $1"
|
||||
},
|
||||
"07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32": {
|
||||
"describe": {
|
||||
@@ -409,28 +417,6 @@
|
||||
},
|
||||
"query": "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company)\n VALUES ($1, $2, $3, 'password', $4, $5, $6)"
|
||||
},
|
||||
"11eb4dd4a2c9b0b759294dde5e8b505c5a4391aa0d8cb629c665711ee0fc04a0": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "logs",
|
||||
"ordinal": 0,
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
null
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3"
|
||||
},
|
||||
"122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
@@ -1495,6 +1481,74 @@
|
||||
},
|
||||
"query": "UPDATE app SET versions = array_append(versions, $1) WHERE id = $2"
|
||||
},
|
||||
"41659beec0e8d6d95258a728e8e4b2b929045a05bd0cfb2293031c0e4a62f04e": {
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Uuid",
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Timestamptz",
|
||||
"Timestamptz",
|
||||
"Bool",
|
||||
"Int8",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Text",
|
||||
"Text",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Varchar",
|
||||
"Text",
|
||||
{
|
||||
"Custom": {
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"script",
|
||||
"preview",
|
||||
"flow",
|
||||
"dependencies",
|
||||
"flowpreview",
|
||||
"script_hub",
|
||||
"identity",
|
||||
"flowdependencies"
|
||||
]
|
||||
},
|
||||
"name": "job_kind"
|
||||
}
|
||||
},
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Bool",
|
||||
"Bool",
|
||||
{
|
||||
"Custom": {
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"python3",
|
||||
"deno",
|
||||
"go",
|
||||
"bash"
|
||||
]
|
||||
},
|
||||
"name": "script_lang"
|
||||
}
|
||||
},
|
||||
"Numeric",
|
||||
"Varchar",
|
||||
"Bool",
|
||||
"Int4"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , logs\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n )\n VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)"
|
||||
},
|
||||
"42e1b5634a9e51247115fa73f85a97b1467c913d012cac9c45bb6a349082dc71": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
@@ -2983,6 +3037,27 @@
|
||||
},
|
||||
"query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, false, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage"
|
||||
},
|
||||
"8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "canceled",
|
||||
"ordinal": 0,
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
false
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"Uuid"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled"
|
||||
},
|
||||
"8c11511a74a41a65f448249a00ebe6964a61d00c2f7b4875a55e64741bf1f0ca": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
@@ -3857,73 +3932,6 @@
|
||||
},
|
||||
"query": "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3"
|
||||
},
|
||||
"a46d97b0c65bbbcc29022c2d010a4853541987a58f0098f9c2e3fd6e40b11a52": {
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"nullable": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Uuid",
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Timestamptz",
|
||||
"Timestamptz",
|
||||
"Bool",
|
||||
"Int8",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Text",
|
||||
"Text",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Varchar",
|
||||
"Text",
|
||||
{
|
||||
"Custom": {
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"script",
|
||||
"preview",
|
||||
"flow",
|
||||
"dependencies",
|
||||
"flowpreview",
|
||||
"script_hub",
|
||||
"identity",
|
||||
"flowdependencies"
|
||||
]
|
||||
},
|
||||
"name": "job_kind"
|
||||
}
|
||||
},
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Bool",
|
||||
"Bool",
|
||||
{
|
||||
"Custom": {
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"python3",
|
||||
"deno",
|
||||
"go",
|
||||
"bash"
|
||||
]
|
||||
},
|
||||
"name": "script_lang"
|
||||
}
|
||||
},
|
||||
"Numeric",
|
||||
"Varchar",
|
||||
"Bool"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , logs\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n )\n VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)"
|
||||
},
|
||||
"a6145b0482c9e5da245059a80b1563cad20318fd2dd8aef33f9ca97de1826b8b": {
|
||||
"describe": {
|
||||
"columns": [],
|
||||
@@ -4371,6 +4379,34 @@
|
||||
},
|
||||
"query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at FROM token WHERE email = $1\n ORDER BY created_at DESC"
|
||||
},
|
||||
"b69f747eae8b13a0a8d6914a3a7ad322554fcfe62cd28c5f0a475f18dd770d61": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"name": "logs",
|
||||
"ordinal": 0,
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"name": "mem_peak",
|
||||
"ordinal": 1,
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"nullable": [
|
||||
null,
|
||||
true
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
}
|
||||
},
|
||||
"query": "SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3"
|
||||
},
|
||||
"b7dd791cd69748ef51b7520f505c0c8bb1b4014a273476eddfecf1ab658a18b4": {
|
||||
"describe": {
|
||||
"columns": [
|
||||
|
||||
@@ -3292,6 +3292,8 @@ paths:
|
||||
type: boolean
|
||||
new_logs:
|
||||
type: string
|
||||
mem_peak:
|
||||
type: integer
|
||||
|
||||
/w/{workspace}/jobs/completed/get/{id}:
|
||||
get:
|
||||
@@ -4790,6 +4792,8 @@ components:
|
||||
type: string
|
||||
visible_to_owner:
|
||||
type: boolean
|
||||
mem_peak:
|
||||
type: integer
|
||||
required:
|
||||
- id
|
||||
- running
|
||||
@@ -4876,6 +4880,8 @@ components:
|
||||
type: string
|
||||
visible_to_owner:
|
||||
type: boolean
|
||||
mem_peak:
|
||||
type: integer
|
||||
required:
|
||||
- id
|
||||
- created_by
|
||||
|
||||
@@ -258,6 +258,8 @@ pub struct CompletedJob {
|
||||
pub is_skipped: bool,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
@@ -487,6 +489,7 @@ async fn list_jobs(
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"suspend",
|
||||
"mem_peak",
|
||||
],
|
||||
);
|
||||
let sqlc = list_completed_jobs_query(
|
||||
@@ -521,6 +524,7 @@ async fn list_jobs(
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"null as suspend",
|
||||
"mem_peak",
|
||||
],
|
||||
);
|
||||
let sql = format!(
|
||||
@@ -998,6 +1002,7 @@ struct UnifiedJob {
|
||||
email: String,
|
||||
visible_to_owner: bool,
|
||||
suspend: Option<i32>,
|
||||
mem_peak: Option<i32>,
|
||||
}
|
||||
|
||||
impl From<UnifiedJob> for Job {
|
||||
@@ -1032,6 +1037,7 @@ impl From<UnifiedJob> for Job {
|
||||
is_skipped: uj.is_skipped,
|
||||
email: uj.email,
|
||||
visible_to_owner: uj.visible_to_owner,
|
||||
mem_peak: uj.mem_peak,
|
||||
}),
|
||||
"QueuedJob" => Job::QueuedJob(QueuedJob {
|
||||
workspace_id: uj.workspace_id,
|
||||
@@ -1064,6 +1070,7 @@ impl From<UnifiedJob> for Job {
|
||||
email: uj.email,
|
||||
visible_to_owner: uj.visible_to_owner,
|
||||
suspend: uj.suspend,
|
||||
mem_peak: uj.mem_peak,
|
||||
}),
|
||||
t => panic!("job type {} not valid", t),
|
||||
}
|
||||
@@ -1582,6 +1589,7 @@ pub struct JobUpdate {
|
||||
pub running: Option<bool>,
|
||||
pub completed: Option<bool>,
|
||||
pub new_logs: Option<String>,
|
||||
pub mem_peak: Option<i32>,
|
||||
}
|
||||
|
||||
async fn get_job_update(
|
||||
@@ -1591,8 +1599,8 @@ async fn get_job_update(
|
||||
) -> error::JsonResult<JobUpdate> {
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let logs = query_scalar!(
|
||||
"SELECT substr(logs, $1) as logs FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
let record = sqlx::query!(
|
||||
"SELECT substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
|
||||
log_offset,
|
||||
&w_id,
|
||||
&id
|
||||
@@ -1600,12 +1608,13 @@ async fn get_job_update(
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
|
||||
if let Some(logs) = logs {
|
||||
if let Some(record) = record {
|
||||
tx.commit().await?;
|
||||
Ok(Json(JobUpdate {
|
||||
running: if !running { Some(true) } else { None },
|
||||
completed: None,
|
||||
new_logs: logs,
|
||||
new_logs: record.logs,
|
||||
mem_peak: record.mem_peak,
|
||||
}))
|
||||
} else {
|
||||
let logs = query_scalar!(
|
||||
@@ -1623,6 +1632,7 @@ async fn get_job_update(
|
||||
running: Some(false),
|
||||
completed: Some(true),
|
||||
new_logs: logs,
|
||||
mem_peak: record.map(|r| r.mem_peak).flatten(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -1740,6 +1750,7 @@ async fn list_completed_jobs(
|
||||
"is_skipped",
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"mem_peak",
|
||||
],
|
||||
)
|
||||
.sql()?;
|
||||
|
||||
@@ -671,6 +671,8 @@ pub struct QueuedJob {
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suspend: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
|
||||
@@ -77,6 +77,12 @@ pub async fn add_completed_job(
|
||||
None
|
||||
};
|
||||
|
||||
let mem_peak = sqlx::query_scalar!("SELECT mem_peak FROM queue WHERE id = $1", &queued_job.id)
|
||||
.fetch_optional(db)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten();
|
||||
let mut tx = db.begin().await?;
|
||||
let job_id = queued_job.id.clone();
|
||||
sqlx::query!(
|
||||
@@ -109,9 +115,10 @@ pub async fn add_completed_job(
|
||||
, language
|
||||
, email
|
||||
, visible_to_owner
|
||||
, mem_peak
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28)
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
|
||||
ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
|
||||
queued_job.workspace_id,
|
||||
queued_job.id,
|
||||
@@ -140,7 +147,8 @@ pub async fn add_completed_job(
|
||||
queued_job.language: ScriptLang,
|
||||
duration: Option<i64>,
|
||||
queued_job.email,
|
||||
queued_job.visible_to_owner
|
||||
queued_job.visible_to_owner,
|
||||
mem_peak
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await
|
||||
|
||||
@@ -35,8 +35,7 @@ use tokio::{
|
||||
io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
|
||||
process::{Child, Command},
|
||||
sync::{
|
||||
mpsc::{self, Sender},
|
||||
oneshot, watch,
|
||||
mpsc::{self, Sender}, watch,
|
||||
},
|
||||
time::{interval, sleep, Instant, MissedTickBehavior},
|
||||
};
|
||||
@@ -1345,7 +1344,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, build_go).await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, build_go, false).await?;
|
||||
|
||||
Command::new(nsjail_path)
|
||||
.current_dir(job_dir)
|
||||
@@ -1371,7 +1370,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child, !disable_nsjail).await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
@@ -1450,7 +1449,7 @@ async fn handle_bash_job(
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child, !disable_nsjail).await?;
|
||||
//for now bash jobs have an empty result object
|
||||
Ok(serde_json::json!(logs
|
||||
.lines()
|
||||
@@ -1614,7 +1613,7 @@ run().catch(async (e) => {{
|
||||
}
|
||||
.instrument(trace_span!("create_deno_jail"))
|
||||
.await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child, !disable_nsjail).await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
@@ -1888,7 +1887,7 @@ mount {{
|
||||
.spawn()?
|
||||
};
|
||||
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child).await?;
|
||||
handle_child(&job.id, db, logs, timeout, *max_log_size, child, !disable_nsjail).await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
@@ -2194,7 +2193,7 @@ async fn pip_compile(
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(job_id, db, logs, timeout, *max_log_size, child)
|
||||
handle_child(job_id, db, logs, timeout, *max_log_size, child, false)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
let path_lock = format!("{job_dir}/requirements.txt");
|
||||
@@ -2230,7 +2229,7 @@ async fn install_go_dependencies(
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
handle_child(job_id, db, logs, timeout, max_log_size, child).await?;
|
||||
handle_child(job_id, db, logs, timeout, max_log_size, child, false).await?;
|
||||
}
|
||||
let child = Command::new(go_path)
|
||||
.current_dir(job_dir)
|
||||
@@ -2238,7 +2237,7 @@ async fn install_go_dependencies(
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(job_id, db, logs, timeout, max_log_size, child)
|
||||
handle_child(job_id, db, logs, timeout, max_log_size, child, false)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
|
||||
@@ -2308,6 +2307,30 @@ async fn get_reserved_variables(
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
|
||||
if pid.is_none() {
|
||||
return -1
|
||||
}
|
||||
let pid = if nsjail {
|
||||
// This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
|
||||
// Ideally, we would get the number from fork() itself. This works in MOST cases.
|
||||
pid.unwrap() + 1
|
||||
} else {
|
||||
pid.unwrap()
|
||||
};
|
||||
|
||||
if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
|
||||
let mut lines = BufReader::new(file).lines();
|
||||
while let Some(line) = lines.next_line().await.unwrap_or(None) {
|
||||
if line.starts_with("VmPeak:") {
|
||||
return line.split_whitespace().nth(1).and_then(|s| s.parse::<i32>().ok()).unwrap_or(-1);
|
||||
};
|
||||
}
|
||||
-2
|
||||
} else {
|
||||
-3
|
||||
}
|
||||
}
|
||||
/// - wait until child exits and return with exit status
|
||||
/// - read lines from stdout and stderr and append them to the "queue"."logs"
|
||||
/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
|
||||
@@ -2322,17 +2345,17 @@ async fn handle_child(
|
||||
timeout: i32,
|
||||
max_log_size: i64,
|
||||
mut child: Child,
|
||||
nsjail: bool,
|
||||
) -> error::Result<()> {
|
||||
let timeout = Duration::from_secs(u64::try_from(timeout).expect("invalid timeout"));
|
||||
let ping_interval = Duration::from_secs(5);
|
||||
let cancel_check_interval = Duration::from_millis(500);
|
||||
let update_job_interval = Duration::from_millis(500);
|
||||
let write_logs_delay = Duration::from_millis(500);
|
||||
|
||||
if let Some(pid) = child.id() {
|
||||
let pid = child.id();
|
||||
if let Some(pid) = pid {
|
||||
//set the highest oom priority
|
||||
let mut file = File::create(format!("/proc/{pid}/oom_score_adj")).await?;
|
||||
let _ = file.write_all(b"1000").await;
|
||||
tracing::info!("set oom_score_adj to 1000 for pid {}", pid);
|
||||
} else {
|
||||
tracing::info!("could not get child pid");
|
||||
}
|
||||
@@ -2345,17 +2368,20 @@ async fn handle_child(
|
||||
|
||||
/* the cancellation future is polled on by `wait_on_child` while
|
||||
* waiting for the child to exit normally */
|
||||
let cancel_check = async {
|
||||
let update_job = async {
|
||||
let db = db.clone();
|
||||
|
||||
let mut interval = interval(cancel_check_interval);
|
||||
let mut interval = interval(update_job_interval);
|
||||
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
|
||||
loop {
|
||||
tokio::select!(
|
||||
_ = rx.recv() => break,
|
||||
_ = interval.tick() => {
|
||||
if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job_id)
|
||||
let mem_peak = get_mem_peak(pid, nsjail).await;
|
||||
tracing::info!("{job_id} still running. mem peak: {}kB", mem_peak);
|
||||
let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
|
||||
if sqlx::query_scalar!("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id)
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
.map(|v| Some(true) == v)
|
||||
@@ -2385,7 +2411,7 @@ async fn handle_child(
|
||||
biased;
|
||||
result = child.wait() => return result.map(Ok),
|
||||
Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
|
||||
_ = cancel_check => KillReason::Cancelled,
|
||||
_ = update_job => KillReason::Cancelled,
|
||||
_ = sleep(timeout) => KillReason::Timeout,
|
||||
};
|
||||
tx.send(()).await.expect("rx should never be dropped");
|
||||
@@ -2508,31 +2534,8 @@ async fn handle_child(
|
||||
}
|
||||
}.instrument(trace_span!("child_lines"));
|
||||
|
||||
/* a stream updating "queue"."last_ping" at an interval */
|
||||
|
||||
let (kill_tx, mut kill_rx) = oneshot::channel::<()>();
|
||||
|
||||
let mut interval = interval(ping_interval);
|
||||
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
|
||||
let db1 = db.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(err) = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
|
||||
.execute(&db1)
|
||||
.await
|
||||
{
|
||||
tracing::error!(%job_id, %err, "error setting last ping for job {job_id}: {err}");
|
||||
};
|
||||
},
|
||||
_ = (&mut kill_rx) => return,
|
||||
}
|
||||
}
|
||||
});
|
||||
let (wait_result, _) = tokio::join!(wait_on_child, lines);
|
||||
kill_tx.send(()).expect("send should always work");
|
||||
|
||||
match wait_result {
|
||||
_ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
|
||||
@@ -2844,7 +2847,7 @@ async fn handle_python_reqs(
|
||||
.spawn()?
|
||||
};
|
||||
|
||||
let child = handle_child(&job.id, db, logs, timeout, *max_log_size, child).await;
|
||||
let child = handle_child(&job.id, db, logs, timeout, *max_log_size, child, false).await;
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
job_id = %job.id,
|
||||
|
||||
@@ -2,14 +2,15 @@
|
||||
import type { Job } from '$lib/gen'
|
||||
import JobStatus from '$lib/components/JobStatus.svelte'
|
||||
import Icon from 'svelte-awesome'
|
||||
import { displayDaysAgo } from '$lib/utils'
|
||||
import { displayDate, displayDaysAgo } from '$lib/utils'
|
||||
import {
|
||||
faCalendar,
|
||||
faClock,
|
||||
faRobot,
|
||||
faScroll,
|
||||
faUser,
|
||||
faBarsStaggered
|
||||
faBarsStaggered,
|
||||
faMemory
|
||||
} from '@fortawesome/free-solid-svg-icons'
|
||||
import ScheduleEditor from './ScheduleEditor.svelte'
|
||||
import { onDestroy, onMount } from 'svelte'
|
||||
@@ -39,16 +40,31 @@
|
||||
<Icon class="text-gray-700" data={faClock} scale={SMALL_ICON_SCALE} /><span
|
||||
class="mx-2 text-2xs text-gray-600"
|
||||
>
|
||||
{#key time}
|
||||
Received job {displayDaysAgo(job.created_at ?? '')}
|
||||
{/key}</span
|
||||
{#if job['success'] != undefined}
|
||||
Received job: {displayDate(job.created_at ?? '')}
|
||||
{:else}
|
||||
{#key time}
|
||||
Received job {displayDaysAgo(job.created_at ?? '')}
|
||||
{/key}
|
||||
{/if}</span
|
||||
>
|
||||
</div>
|
||||
{#if job && 'started_at' in job && job.started_at}
|
||||
<div>
|
||||
<Icon class="text-gray-700" data={faClock} scale={SMALL_ICON_SCALE} /><span class="mx-2">
|
||||
{#key time}
|
||||
Started {displayDaysAgo(job.started_at ?? '')}{/key}</span
|
||||
{#if job['success'] != undefined}
|
||||
Started: {displayDate(job.started_at ?? '')}
|
||||
{:else}
|
||||
{#key time}
|
||||
Started {displayDaysAgo(job.started_at ?? '')}{/key}
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
{#if job && job['mem_peak']}
|
||||
<div>
|
||||
<Icon class="text-gray-700" data={faMemory} scale={SMALL_ICON_SCALE} /><span class="mx-2">
|
||||
Mem peak: {(job['mem_peak'] / 1024).toPrecision(5)}MB</span
|
||||
>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
export let content: string | undefined
|
||||
export let isLoading: boolean
|
||||
export let duration: number | undefined = undefined
|
||||
export let mem: number | undefined = undefined
|
||||
|
||||
let scroll = true
|
||||
let div: HTMLElement | null = null
|
||||
@@ -50,6 +51,11 @@
|
||||
{:else if duration}
|
||||
<span class="absolute text-xs text-gray-500 top-2 left-2">took {duration}ms</span>
|
||||
{/if}
|
||||
{#if mem}
|
||||
<span class="absolute text-xs text-gray-500 top-2 left-36"
|
||||
>mem peak: {(mem / 1024).toPrecision(4)}MB</span
|
||||
>
|
||||
{/if}
|
||||
<pre class="whitespace-pre-wrap break-words bg-gray-50 text-xs w-full p-2"
|
||||
>{#if content}<span>{content}</span>{:else if !isLoading}<span class="text-gray-600"
|
||||
>No logs are available yet</span
|
||||
|
||||
@@ -146,6 +146,9 @@
|
||||
if (previewJobUpdates.new_logs) {
|
||||
job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
|
||||
}
|
||||
if (previewJobUpdates.mem_peak) {
|
||||
job.mem_peak = previewJobUpdates.mem_peak
|
||||
}
|
||||
if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
|
||||
job = await JobService.getJob({ workspace: workspace!, id })
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@
|
||||
<Pane class="relative">
|
||||
<LogViewer
|
||||
duration={previewJob?.['duration_ms']}
|
||||
mem={previewJob?.['mem_peak']}
|
||||
content={previewJob?.logs}
|
||||
isLoading={previewIsLoading}
|
||||
/>
|
||||
|
||||
Reference in New Issue
Block a user