Files
windmill/backend/migrations/20260311100000_audit_partitioning.up.sql
Ruben Fiszel 2aef01d18c feat: partition audit log table by day with configurable retention (#8292)
* feat: partition audit log table by day with configurable retention

Introduce daily range partitioning for audit logs to replace expensive
DELETE-based retention with instant DROP TABLE per partition.

- Create `audit_partitioned` table alongside existing `audit` table
- New inserts go to `audit_partitioned`, reads UNION ALL both tables
- Monitor creates future partitions and drops expired ones
- Add `audit_log_retention_days` instance setting (default 365 days)
- Old `audit` table empties naturally via existing DELETE cleanup

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add audit log retention setting to Core instance settings UI

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: bump audit partitioning migration timestamp to avoid collision

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore: update ee-repo-ref.txt for audit partitioning

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add RLS/grants to audit_partitioned, run partition mgmt hourly, CE default 14d

- Add grants for windmill_user/windmill_admin and all 5 RLS policies
- Move manage_audit_partitions to hourly via should_run(120)
- Default retention: 14 days CE, 365 days EE
- Download JSON button is now icon-only

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: address code review — quote SQL identifiers, add workspace index, deduplicate retention logic

- Quote partition names in dynamic SQL for defense in depth
- Add idx_audit_partitioned_workspace(workspace_id, timestamp DESC) index
- Extract audit_log_retention_days() helper to deduplicate retention logic

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore: update ee-repo-ref for audit insert error handling

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore: update ee-repo-ref to cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64

This commit updates the EE repository reference after PR #450 was merged in windmill-ee-private.

Previous ee-repo-ref: f09284bb257d461bcbe3c50fe31eb6f1e7eafee5

New ee-repo-ref: cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64

Automated by sync-ee-ref workflow.

* fix: create audit partitions on startup in initial_load

Ensures partitions exist before any requests arrive, closing the gap
between server start and the first hourly monitor run.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
2026-03-11 00:10:29 +00:00

59 lines
2.4 KiB
SQL

-- Create a new daily-partitioned audit table alongside the existing one.
-- New inserts go to audit_partitioned; reads UNION ALL both tables.
-- The old audit table empties out naturally via retention cleanup.
CREATE TABLE audit_partitioned (
workspace_id VARCHAR(50) NOT NULL,
id BIGINT NOT NULL DEFAULT nextval('audit_id_seq'),
timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
username VARCHAR(255) NOT NULL,
operation VARCHAR(50) NOT NULL,
action_kind ACTION_KIND NOT NULL,
resource VARCHAR(255),
parameters JSONB,
email VARCHAR(255),
span VARCHAR(255),
PRIMARY KEY (id, timestamp)
) PARTITION BY RANGE (timestamp);
-- Create daily partitions for today + 3 days
DO $$
DECLARE
curr_date DATE := CURRENT_DATE;
end_date DATE := CURRENT_DATE + INTERVAL '3 days';
BEGIN
WHILE curr_date <= end_date LOOP
EXECUTE format(
'CREATE TABLE %I PARTITION OF audit_partitioned FOR VALUES FROM (%L) TO (%L)',
'audit_' || to_char(curr_date, 'YYYYMMDD'),
curr_date,
curr_date + INTERVAL '1 day'
);
curr_date := curr_date + INTERVAL '1 day';
END LOOP;
END $$;
-- Indexes (auto-propagated to all current and future partitions)
CREATE INDEX ix_audit_partitioned_timestamps ON audit_partitioned (timestamp DESC);
CREATE INDEX idx_audit_partitioned_workspace ON audit_partitioned (workspace_id, timestamp DESC);
CREATE INDEX idx_audit_partitioned_recent_login_activities
ON audit_partitioned (timestamp, username)
WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');
-- Grants (match the old audit table)
GRANT ALL ON audit_partitioned TO windmill_user;
GRANT ALL ON audit_partitioned TO windmill_admin;
-- RLS (match the old audit table)
ALTER TABLE audit_partitioned ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON audit_partitioned FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_own ON audit_partitioned FOR ALL TO windmill_user
USING ((username)::text = current_setting('session.user'::text));
CREATE POLICY schedule ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((username)::text ~~ 'schedule-%'::text);
CREATE POLICY schedule_audit ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((parameters ->> 'end_user'::text) ~~ 'schedule-%'::text);
CREATE POLICY webhook ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((username)::text ~~ 'webhook-%'::text);