kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement quiet mode to filter verbose logs (#7478)

Browse Source 
* feat: implement quiet mode to filter verbose logs

Add QUIET_MODE environment variable (set to "true" or "1" to enable) that:
- Filters logs with the "windmill_verbose" target via tracing_init
- Increases frequency of periodic logs by 10x when enabled
- Marks per-job worker logs as verbose so they are suppressed in quiet mode

Key changes:
- Add QUIET_MODE lazy_static and VERBOSE_TARGET constant to tracing_init.rs
- Create create_targets_filter() helper that filters windmill_verbose target
- Mark periodic "still running" and "memory snapshot" logs with VERBOSE_TARGET
- Mark per-job "fetched job", "started handling", "job finished" logs as verbose
- Increase memory snapshot interval from every 10 ticks to every 100 ticks in quiet mode
- Increase "still running" log interval from every tick to every 10 ticks in quiet mode
- Add quiet mode notification in job logs

Closes #7477

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>

* Update tracing_init.rs

* Update worker.rs

* Remove target from tracing info logs

---------

Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
claude[bot]
2026-01-06 18:38:58 +00:00
committed by GitHub
parent dd26c7d00f
commit 6807fc6a3c
3 changed files with 42 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
backend/windmill-common/src/tracing_init.rs
View File

@@ -38,6 +38,25 @@ fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::
lazy_static::lazy_static! {
pub static ref JSON_FMT: bool = std::env::var("JSON_FMT").map(|x| x == "true").unwrap_or(false);
pub static ref QUIET_MODE: bool = std::env::var("QUIET").map(|x| x == "true" || x == "1").unwrap_or(false);
}
/// Target name for verbose logs that should be filtered in quiet mode.
/// Use `tracing::info!(target: windmill_common::tracing_init::VERBOSE_TARGET, ...)` for logs that should be suppressed in quiet mode.
pub const VERBOSE_TARGET: &str = "windmill_verbose";
/// Creates a Targets filter that optionally filters out verbose logs when quiet mode is enabled.
fn create_targets_filter(default_env_filter: LevelFilter) -> Targets {
let targets = Targets::new()
.with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF);
if *QUIET_MODE {
targets
.with_target(VERBOSE_TARGET, tracing::level_filters::LevelFilter::OFF)
.with_default(default_env_filter)
} else {
targets.with_default(default_env_filter)
}
}
pub const LOGS_SERVICE: &str = "logs/services/";
@@ -156,22 +175,14 @@ pub fn initialize_tracing(
.with_writer(std::io::stdout)
.flatten_event(true)
.with_filter(stdout_env_filter)
.with_filter(
Targets::new()
.with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF)
.with_default(default_env_filter),
);
.with_filter(create_targets_filter(default_env_filter));
// File layer with its own filter
let file_layer = json_layer()
.with_writer(log_file_writer)
.flatten_event(true)
.with_filter(file_env_filter)
.with_filter(
Targets::new()
.with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF)
.with_default(default_env_filter),
);
.with_filter(create_targets_filter(default_env_filter));
base_layer
.with(stdout_layer)
@@ -188,11 +199,7 @@ pub fn initialize_tracing(
.with_line_number(true)
.with_target(false)
.with_filter(stdout_env_filter)
.with_filter(
Targets::new()
.with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF)
.with_default(default_env_filter),
);
.with_filter(create_targets_filter(default_env_filter));
// File layer with its own filter
let file_layer = compact_layer()
@@ -202,11 +209,7 @@ pub fn initialize_tracing(
.with_line_number(true)
.with_target(false)
.with_filter(file_env_filter)
.with_filter(
Targets::new()
.with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF)
.with_default(default_env_filter),
);
.with_filter(create_targets_filter(default_env_filter));
base_layer
.with(stdout_layer)

16
backend/windmill-worker/src/handle_child.rs
View File

@@ -58,6 +58,8 @@ use crate::job_logger_oss::process_streaming_log_lines;
use crate::worker_utils::{ping_job_status, update_worker_ping_from_job};
use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
use windmill_common::tracing_init::{QUIET_MODE, VERBOSE_TARGET};
lazy_static::lazy_static! {
pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
pub static ref OTEL_JOB_LOGS: bool = std::env::var("OTEL_JOB_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
@@ -702,9 +704,11 @@ where
tokio::select!(
_ = rx.recv() => break,
_ = interval.tick() => {
// update the last_ping column every 5 seconds
// update the last_ping column every 5 seconds (or 50 seconds in quiet mode)
i+=1;
if i == 1 || i % 10 == 0 {
// In quiet mode, emit memory snapshot logs 10x less frequently
let memory_snapshot_interval = if *QUIET_MODE { 100 } else { 10 };
if i == 1 || i % memory_snapshot_interval == 0 {
let memory_usage = get_worker_memory_usage();
let wm_memory_usage = get_windmill_memory_usage();
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
@@ -719,7 +723,11 @@ where
if current_mem > *mem_peak {
*mem_peak = current_mem
}
tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
// In quiet mode, emit "still running" logs 10x less frequently
let still_running_interval = if *QUIET_MODE { 10 } else { 1 };
if i % still_running_interval == 0 {
tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
}
let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
@@ -778,7 +786,7 @@ where
},
);
}
tracing::info!("job {job_id} finished");
tracing::info!(target: VERBOSE_TARGET, "job {job_id} finished");
UpdateJobPollingExit::Done(canceled_by_ref.clone())
}

9
backend/windmill-worker/src/worker.rs
View File

@@ -77,6 +77,7 @@ use windmill_common::{
flows::FlowNodeId,
jobs::JobKind,
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
tracing_init::{QUIET_MODE, VERBOSE_TARGET},
utils::StripPath,
worker::{CLOUD_HOSTED, NO_LOGS, WORKER_CONFIG, WORKER_GROUP},
DB, IS_READY,
@@ -1088,7 +1089,7 @@ pub fn start_interactive_worker_shell(
match pulled_job {
Ok(Some(job)) => {
tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
tracing::debug!(target: VERBOSE_TARGET, worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
let job_dir = create_job_dir(&worker_dir, job.id).await;
#[cfg(feature = "benchmark")]
let mut bench = windmill_common::bench::BenchmarkIter::new();
@@ -1956,7 +1957,7 @@ pub async fn run_worker(
last_executed_job = None;
jobs_executed += 1;
tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
tracing::debug!(target: VERBOSE_TARGET, worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
if matches!(
job.kind,
@@ -2890,6 +2891,10 @@ pub async fn handle_queued_job(
logs.push_str("Logs are 10x less frequent for this worker\n");
}
if *QUIET_MODE {
logs.push_str("Quiet mode enabled: verbose service logs are suppressed\n");
}
#[cfg(not(feature = "enterprise"))]
if let Connection::Sql(db) = conn {
if (job.concurrent_limit.is_some() ||