From 275d25366c24be4f1e97d262071396bd0e634cfb Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:12:40 +0200 Subject: [PATCH 1/9] style(memtrack): disallow short functions on one line and reformat Set AllowShortFunctionsOnASingleLine: None in .clang-format and apply it, reformatting the allocator uprobe macros accordingly. --- crates/memtrack/.clang-format | 1 + crates/memtrack/src/ebpf/c/allocator.h | 34 ++++++++++++---------- crates/memtrack/src/ebpf/c/utils/variant.h | 8 +++-- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/crates/memtrack/.clang-format b/crates/memtrack/.clang-format index b687ea407..4755a3406 100644 --- a/crates/memtrack/.clang-format +++ b/crates/memtrack/.clang-format @@ -4,3 +4,4 @@ BasedOnStyle: Google PointerAlignment: Left ColumnLimit: 100 IndentWidth: 4 +AllowShortFunctionsOnASingleLine: None diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index bf7f7e850..9a4cc2387 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,22 +5,24 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC(UPROBE_SEC) \ - int uprobe_##name(struct pt_regs* ctx) { return store_param(&name##_arg, arg_expr); } \ - SEC(URETPROBE_SEC) \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC(UPROBE_SEC) \ + int uprobe_##name(struct pt_regs* ctx) { \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC(URETPROBE_SEC) \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ diff --git a/crates/memtrack/src/ebpf/c/utils/variant.h b/crates/memtrack/src/ebpf/c/utils/variant.h index 036c63ba4..e7222de6b 100644 --- a/crates/memtrack/src/ebpf/c/utils/variant.h +++ b/crates/memtrack/src/ebpf/c/utils/variant.h @@ -77,9 +77,13 @@ static __always_inline struct task_ids current_task_ids(void) { return ids; } -static __always_inline __u32 current_tgid(void) { return current_task_ids().tgid; } +static __always_inline __u32 current_tgid(void) { + return current_task_ids().tgid; +} -static __always_inline __u32 current_tid(void) { return current_task_ids().tid; } +static __always_inline __u32 current_tid(void) { + return current_task_ids().tid; +} /* Thread-group id of an arbitrary task in the configured namespace. * From a615d31a5533b1875163eed2b8a7f409d870c30f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 16:30:09 +0200 Subject: [PATCH 2/9] feat(memtrack): emit fork/exec/exit lifecycle events A forked child's inherited RSS is invisible to rss_stat: the fork-time counter copies fire outside the child's context, and anon COW faults are counter-neutral, so a child that only touches inherited memory never reports anything on its own. A fork event carrying the parent pid lets consumers seed the child from the parent's last absolutes; exec and exit mark where the address space is replaced or torn down. --- crates/memtrack/src/ebpf/c/event.h | 8 ++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/rss.bpf.h | 82 +++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 43 +++++++--- .../src/ebpf/c/utils/process_tracking.h | 23 ------ crates/memtrack/src/ebpf/events.rs | 50 ++++++++--- crates/memtrack/src/ebpf/memtrack/tracking.rs | 8 +- crates/memtrack/src/ebpf/tracker.rs | 5 +- crates/memtrack/src/main.rs | 9 +- .../src/artifacts/memtrack/mod.rs | 5 ++ 10 files changed, 179 insertions(+), 55 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/rss.bpf.h diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index ff41f0bea..6af3bcd4f 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -9,6 +9,9 @@ #define EVENT_TYPE_MMAP 6 #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 +#define EVENT_TYPE_FORK 9 +#define EVENT_TYPE_EXEC 10 +#define EVENT_TYPE_EXIT 11 /* Common header shared by all event types */ struct event_header { @@ -45,6 +48,11 @@ struct event { uint64_t addr; /* address of mapping */ uint64_t size; /* size of mapping */ } mmap; + + /* Process lifecycle events (fork carries the parent; exec/exit have no payload) */ + struct { + uint32_t parent_pid; + } fork; } data; }; diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 1c5ad8d12..e46a144bf 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "rss.bpf.h" #include "utils/event_helpers.h" #include "utils/map_helpers.h" #include "utils/process_tracking.h" diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h new file mode 100644 index 000000000..43ef5555c --- /dev/null +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -0,0 +1,82 @@ +#ifndef __RSS_BPF_H__ +#define __RSS_BPF_H__ + +#include "event.h" +#include "utils/event_helpers.h" +#include "utils/process_tracking.h" + +/* == Process lifecycle events == + * + * FORK lets userland seed a child's RSS from its parent at fork time: the + * kernel copies the mm counters during dup_mmap, but those updates fire + * rss_stat out of the child's context and anon COW faults are + * counter-neutral, so a child that only touches inherited memory never + * reports its RSS on its own. EXEC and EXIT mark the points where the + * address space is replaced or torn down, so userland resets to zero. + */ + +#define CLONE_THREAD 0x00010000 + +SEC("tracepoint/task/task_newtask") +int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { + if (ctx->clone_flags & CLONE_THREAD) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 parent_pid = tid >> 32; + if (!is_tracked(parent_pid)) { + return 0; + } + + /* Register the child here rather than on sched_process_fork: that + * tracepoint fires for CLONE_THREAD too and carries only raw task pids, + * which would fill the tracking maps with thread tids that no exit path + * removes (group death deletes only the tgid). task_newtask fires before + * wake_up_new_task, so registration precedes any event from the child. */ + __u32 child_pid = ctx->pid; + track_child(child_pid, parent_pid); + + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); +} + +SEC("tracepoint/sched/sched_process_exec") +int tracepoint_sched_process_exec(void* ctx) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXEC, {}); +} + +SEC("tracepoint/sched/sched_process_exit") +int tracepoint_sched_process_exit(void* ctx) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + /* EXIT marks the death of the whole thread group, not of one thread: the + * leader can pthread_exit while workers keep running, and the last thread + * to exit need not be the leader. do_exit decrements signal->live before + * this tracepoint fires, so live == 0 identifies the dying thread group's + * final exit — but concurrently exiting threads can BOTH read 0, so the + * tracked_pids delete below arbitrates: only the task that wins it emits. */ + struct task_struct* task = bpf_get_current_task_btf(); + if (BPF_CORE_READ(task, signal, live.counter) != 0) { + return 0; + } + + /* Untrack the pid before submitting: lifetime events are gated only on + * is_tracked, so a stale entry would keep streaming events if the kernel + * reuses the pid for an unrelated process. */ + if (bpf_map_delete_elem(&tracked_pids, &pid) != 0) { + return 0; + } + bpf_map_delete_elem(&pids_ppid, &pid); + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXIT, {}); +} + +#endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index 89c8be14f..b35e30c8c 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -40,14 +40,19 @@ static __always_inline __u64* take_param(void* map) { return value; } -#define SUBMIT_EVENT(evt_type, fill_data) \ +/* Submission is split into two classes: + * - lifetime events (rss_stat, rmap, fork/exec/exit): emitted whenever the + * pid is tracked, ignoring the enable toggle. The parser reconstructs + * absolute per-process state from these, so it needs every event from + * process birth — a delta stream that starts mid-life can never recover + * the resident baseline faulted before enable. + * - allocator events (malloc/free/mmap/...): high-volume and only meaningful + * inside a measurement window, so they stay behind is_enabled(). + */ +#define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ struct task_ids ids = current_task_ids(); \ \ - if (!is_tracked(ids.tgid) || !is_enabled()) { \ - return 0; \ - } \ - \ struct event* e = bpf_ringbuf_reserve(&events, sizeof(*e), 0); \ if (!e) { \ __u32 zero = 0; \ @@ -59,7 +64,7 @@ static __always_inline __u64* take_param(void* map) { } \ \ e->header.timestamp = bpf_ktime_get_ns(); \ - e->header.pid = ids.tgid; \ + e->header.pid = owner_pid; \ e->header.tid = ids.tid; \ e->header.event_type = evt_type; \ \ @@ -69,33 +74,47 @@ static __always_inline __u64* take_param(void* map) { return 0; \ } +#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ + { \ + if (!is_enabled()) { \ + return 0; \ + } \ + \ + struct task_ids owner = current_task_ids(); \ + if (!is_tracked(owner.tgid)) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_MALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_CALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); } static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { - SUBMIT_EVENT(EVENT_TYPE_REALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; @@ -103,7 +122,7 @@ static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, } static __always_inline int submit_mmap_event(__u64 addr, __u64 size, __u8 event_type) { - SUBMIT_EVENT(event_type, { + SUBMIT_GATED_EVENT(event_type, { e->data.mmap.addr = addr; e->data.mmap.size = size; }); diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 7b147cbc4..821f3d720 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -44,27 +44,4 @@ static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { bpf_map_update_elem(&pids_ppid, &child_pid, &parent_pid, BPF_ANY); } -/* Record a parent→child fork so the child inherits the parent's tracked state. */ -static __always_inline void follow_fork(__u32 parent_pid, __u32 child_pid) { - if (parent_pid == 0 || child_pid == 0) { - return; - } - if (is_tracked(parent_pid)) { - track_child(child_pid, parent_pid); - } -} - -/* tp_btf rather than a classic tracepoint on two counts: it attaches with - * BPF_RAW_TRACEPOINT_OPEN, so a token can delegate it (perf_event_open() cannot - * be), and its arguments are task_struct pointers rather than a flattened trace - * event, so PIDs resolve through the tracker's namespace. - * - * https://docs.ebpf.io/linux/program-type/BPF_PROG_TYPE_TRACING/ - */ -SEC("tp_btf/sched_process_fork") -int BPF_PROG(tracepoint_sched_fork, struct task_struct* parent, struct task_struct* child) { - follow_fork(task_ns_tgid(parent), task_ns_tgid(child)); - return 0; -} - #endif /* __PROCESS_TRACKING_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index f18216c6e..52ef752a5 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -74,6 +74,14 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.mmap.size, }, ), + EVENT_TYPE_FORK => ( + 0, + MemtrackEventKind::Fork { + parent_pid: event.data.fork.parent_pid as i32, + }, + ), + EVENT_TYPE_EXEC => (0, MemtrackEventKind::Exec), + EVENT_TYPE_EXIT => (0, MemtrackEventKind::Exit), unknown => { panic!("Unknown event type: {unknown}"); } @@ -118,6 +126,12 @@ impl AttachRequest { mod tests { use super::*; + fn event_bytes(event: &bindings::event) -> &[u8] { + unsafe { + std::slice::from_raw_parts(event as *const _ as *const u8, std::mem::size_of_val(event)) + } + } + #[test] fn test_parse_realloc_event() { // Create a mock event with realloc data @@ -130,12 +144,7 @@ mod tests { event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); // Parse and validate: let parsed = parse_event(bytes).unwrap(); @@ -164,12 +173,7 @@ mod tests { event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); // Parse and validate: let parsed = parse_event(bytes).unwrap(); @@ -185,4 +189,26 @@ mod tests { _ => panic!("Expected Malloc event kind"), } } + + #[test] + fn test_parse_fork_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_FORK as u8; + event.header.timestamp = 12345678; + event.header.pid = 1001; + event.header.tid = 2000; + event.data.fork.parent_pid = 1000; + + let bytes = event_bytes(&event); + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1001); + + match parsed.kind { + MemtrackEventKind::Fork { parent_pid } => { + assert_eq!(parent_pid, 1000); + } + _ => panic!("Expected Fork event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 9b3935b65..bf3f635e6 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -3,10 +3,14 @@ use crate::prelude::*; use paste::paste; impl MemtrackBpf { - attach_tracepoint!(sched_fork); + attach_tracepoint!(task_newtask); + attach_tracepoint!(sched_process_exec); + attach_tracepoint!(sched_process_exit); pub fn attach_tracepoints(&mut self) -> Result<()> { - self.attach_sched_fork()?; + self.attach_task_newtask()?; + self.attach_sched_process_exec()?; + self.attach_sched_process_exit()?; Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 094da2eb4..e87c1ceeb 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -74,12 +74,13 @@ impl Tracker { Ok(Session::new(child, rx, poller)) } - /// Enable event tracking in the BPF program + /// Enable allocator-event tracking in the BPF program. Lifetime events + /// (fork/exec/exit) are emitted for tracked pids regardless of this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } - /// Disable event tracking in the BPF program + /// Disable allocator-event tracking in the BPF program pub fn disable_tracking(&self) -> Result<()> { self.bpf.lock().disable_tracking() } diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 8ad48791a..283cff194 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -95,8 +95,9 @@ fn track_command( } })) } else { - // Without IPC, nothing toggles the tracking_enabled map, so events would - // be dropped by the eBPF is_enabled() check. Enable it up front. + // Without IPC, nothing toggles the tracking_enabled map, so allocator + // events would be dropped by the eBPF is_enabled() check. Enable it up + // front. tracker.enable_tracking()?; None }; @@ -135,8 +136,8 @@ fn track_command( let status = session.wait().context("Failed to wait for command")?; debug!("Command exited with status: {status}"); - // Stop event production before draining: the child has exited, so anything - // still arriving is already in the ring buffer. + // Stop allocator-event production before draining: the child has exited, + // so anything still arriving is already in the ring buffer. if let Err(e) = tracker.disable_tracking() { warn!("Failed to disable tracking: {e:#}"); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index ad843b28c..b0e4e4387 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -78,6 +78,11 @@ pub enum MemtrackEventKind { Brk { size: u64, }, + Fork { + parent_pid: pid_t, + }, + Exec, + Exit, } pub struct MemtrackEventStream { From b40cc4b9fc298690529161748bd728c633bf75a5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 16:30:26 +0200 Subject: [PATCH 3/9] feat(memtrack): track RSS via kmem:rss_stat tracepoint Sample the kernel's per-mm resident counter through the kmem:rss_stat tracepoint, emitting absolute byte values per mm member. Adds the EVENT_TYPE_RSS contract, MemtrackEventKind::Rss, the parser arm, and a writer bench case. An rss_stat update from reclaim or another process's madvise fires in the actor's context; track (mm_id, member) -> owning pid so those updates reach the owner. External events may only lower a counter, so stale reads and mm_id collisions cannot invent peaks. --- crates/memtrack/src/ebpf/c/event.h | 6 ++ crates/memtrack/src/ebpf/c/rss.bpf.h | 66 +++++++++++++++++++ crates/memtrack/src/ebpf/events.rs | 32 +++++++++ crates/memtrack/src/ebpf/memtrack/tracking.rs | 4 ++ crates/memtrack/src/ebpf/tracker.rs | 3 +- .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 14 ++++ 7 files changed, 129 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 6af3bcd4f..ccd6ed851 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -12,6 +12,7 @@ #define EVENT_TYPE_FORK 9 #define EVENT_TYPE_EXEC 10 #define EVENT_TYPE_EXIT 11 +#define EVENT_TYPE_RSS 12 /* Common header shared by all event types */ struct event_header { @@ -53,6 +54,11 @@ struct event { struct { uint32_t parent_pid; } fork; + + struct { + int32_t member; + uint64_t size; + } rss; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 43ef5555c..3815c0610 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -5,6 +5,72 @@ #include "utils/event_helpers.h" #include "utils/process_tracking.h" +/* (rss_stat mm_id << 32 | member) -> {owning tgid, last in-context size}. Keyed per + * counter so an external (curr==0) update is attributed only once that mm/member was + * established in-context. An external event may only lower a counter: any size above + * the last in-context value is dropped, so neither a stale/racing reclaim read nor an + * mm_id hash collision with another task can invent a peak. LRU eviction + re-seeding + * on the owner's next in-context event covers hash reuse, so no teardown hook needed. */ +struct rss_owner { + __u32 pid; + __u64 size; +}; +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 40960); + __type(key, __u64); + __type(value, struct rss_owner); +} rss_counter_owner SEC(".maps"); + +static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { + SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { + e->data.rss.member = member; + e->data.rss.size = size; + }); +} + +SEC("tracepoint/kmem/rss_stat") +int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { + if (ctx->member == MM_SWAPENTS) { + return 0; + } + + __u32 cur = bpf_get_current_pid_tgid() >> 32; + __u64 key = ((__u64)ctx->mm_id << 32) | (__u32)ctx->member; + __u64 size = ctx->size; + __u32 owner; + + if (ctx->curr) { + if (!is_tracked(cur)) { + return 0; + } + owner = cur; + struct rss_owner state = {.pid = cur, .size = size}; + bpf_map_update_elem(&rss_counter_owner, &key, &state, BPF_ANY); + } else { + struct rss_owner* found = bpf_map_lookup_elem(&rss_counter_owner, &key); + if (!found) { + return 0; + } + owner = found->pid; + /* The owner's own teardown also presents as curr==0 (current->mm is cleared + * on exit), so drop it. Genuine external actors (reclaim, another process's + * madvise) run in a different task, so cur != owner. */ + if (cur == owner) { + return 0; + } + /* An external actor may only lower a counter. A larger value is a stale + * reclaim read or an mm_id hash collision with another task; dropping it + * keeps the reconstructed peak identical to the in-context timeline. */ + if (size > found->size) { + return 0; + } + found->size = size; + } + + return submit_rss_event(owner, ctx->member, size); +} + /* == Process lifecycle events == * * FORK lets userland seed a child's RSS from its parent at fork time: the diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 52ef752a5..ef5d59b73 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -82,6 +82,13 @@ pub fn parse_event(data: &[u8]) -> Option { ), EVENT_TYPE_EXEC => (0, MemtrackEventKind::Exec), EVENT_TYPE_EXIT => (0, MemtrackEventKind::Exit), + EVENT_TYPE_RSS => ( + 0, + MemtrackEventKind::Rss { + member: event.data.rss.member, + size: event.data.rss.size, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -190,6 +197,31 @@ mod tests { } } + #[test] + fn test_parse_rss_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RSS as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rss.member = 1; + event.data.rss.size = 4096 * 10; + + let bytes = event_bytes(&event); + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0); + + match parsed.kind { + MemtrackEventKind::Rss { member, size } => { + assert_eq!(member, 1); + assert_eq!(size, 4096 * 10); + } + _ => panic!("Expected Rss event kind"), + } + } + #[test] fn test_parse_fork_event() { let mut event: bindings::event = unsafe { std::mem::zeroed() }; diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index bf3f635e6..eec50638e 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -3,6 +3,7 @@ use crate::prelude::*; use paste::paste; impl MemtrackBpf { + attach_tracepoint!(rss_stat); attach_tracepoint!(task_newtask); attach_tracepoint!(sched_process_exec); attach_tracepoint!(sched_process_exit); @@ -11,6 +12,9 @@ impl MemtrackBpf { self.attach_task_newtask()?; self.attach_sched_process_exec()?; self.attach_sched_process_exit()?; + if let Err(e) = self.attach_rss_stat() { + warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); + } Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index e87c1ceeb..732fe50a5 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -75,7 +75,8 @@ impl Tracker { } /// Enable allocator-event tracking in the BPF program. Lifetime events - /// (fork/exec/exit) are emitted for tracked pids regardless of this toggle. + /// (rss_stat, fork/exec/exit) are emitted for tracked pids regardless of + /// this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 62aa24519..17b4f1729 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..8) { + let kind = match rng.gen_range(0..9) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -25,6 +25,10 @@ fn generate_events(n: usize) -> Vec { 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, + 8 => MemtrackEventKind::Rss { + member: rng.gen_range(0..4), + size, + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b0e4e4387..9fe78eb5f 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -83,6 +83,10 @@ pub enum MemtrackEventKind { }, Exec, Exit, + Rss { + member: i32, + size: u64, + }, } pub struct MemtrackEventStream { @@ -121,6 +125,16 @@ mod tests { addr: 0x20, kind: MemtrackEventKind::Free, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 300, + addr: 0, + kind: MemtrackEventKind::Rss { + member: 1, + size: 40960, + }, + }, ]; let artifact = MemtrackArtifact { From 75bed352919754688d4021f478b884b17b17a94f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 16:31:26 +0200 Subject: [PATCH 4/9] feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks Attach fentry hooks on the folio-rmap add/remove functions, emitting signed page-count deltas per MM_* bucket so anon, file, and shmem RSS can be reconstructed over time. Gated behind CODSPEED_MEMTRACK_TRACK_RMAP; the programs stay autoload-off by default so the skeleton loads on any kernel, with the PUD pair (only present since v6.15) gated separately from the core set so rmap still works on older kernels. Adds the EVENT_TYPE_RMAP contract, MemtrackEventKind::Rmap, parser arm, and bench case. Recover the owning pid for rmap events run by another task (reclaim, process_madvise, khugepaged, KSM) from the mm_struct pointer, and maintain the ownership maps across exec and thread-group exit. The same ownership binding also validates external (curr==0) rss_stat updates, so a stale mm can no longer attribute a counter to the wrong pid. --- crates/memtrack/src/ebpf/c/event.h | 7 + crates/memtrack/src/ebpf/c/rss.bpf.h | 262 +++++++++++++++++- crates/memtrack/src/ebpf/events.rs | 33 +++ crates/memtrack/src/ebpf/memtrack/macros.rs | 24 ++ crates/memtrack/src/ebpf/memtrack/mod.rs | 145 +++++++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 21 ++ crates/memtrack/src/ebpf/tracker.rs | 14 +- .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 4 + 9 files changed, 489 insertions(+), 27 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index ccd6ed851..bf0677c93 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -13,6 +13,7 @@ #define EVENT_TYPE_EXEC 10 #define EVENT_TYPE_EXIT 11 #define EVENT_TYPE_RSS 12 +#define EVENT_TYPE_RMAP 13 /* Common header shared by all event types */ struct event_header { @@ -59,6 +60,12 @@ struct event { int32_t member; uint64_t size; } rss; + + struct { + int32_t member; /* MM_* counter index */ + int64_t delta; + uint64_t addr; + } rmap; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 3815c0610..c0d3925a3 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -5,14 +5,13 @@ #include "utils/event_helpers.h" #include "utils/process_tracking.h" -/* (rss_stat mm_id << 32 | member) -> {owning tgid, last in-context size}. Keyed per - * counter so an external (curr==0) update is attributed only once that mm/member was - * established in-context. An external event may only lower a counter: any size above - * the last in-context value is dropped, so neither a stale/racing reclaim read nor an - * mm_id hash collision with another task can invent a peak. LRU eviction + re-seeding - * on the owner's next in-context event covers hash reuse, so no teardown hook needed. */ +/* (rss_stat mm_id << 32 | member) -> {owning tgid, its mm when seeded, last + * in-context size}. mm_id is a hash, so an external (curr==0) update is accepted + * only while pid_mm still binds the owner to the seeded mm, and only to lower + * the counter — collisions and stale reads must not invent a peak. */ struct rss_owner { __u32 pid; + __u64 mm; __u64 size; }; struct { @@ -22,6 +21,65 @@ struct { __type(value, struct rss_owner); } rss_counter_owner SEC(".maps"); +/* Foreign-actor rmap attribution: rmap events run by a task other than the mm's + * owner (kswapd reclaim, another process's process_madvise, khugepaged, KSM, + * uffd) carry no owning-pid context, so mm_owner recovers it from the mm_struct + * pointer. pid_mm is the inverse, letting exec and exit remove an entry by + * value; attribution requires both to agree, so a stale mm fails closed. + * + * pid_mm must not use LRU eviction: losing the inverse binding would leave exec + * and exit unable to remove the forward entry. */ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 10240); + __type(key, __u64); + __type(value, __u32); +} mm_owner SEC(".maps"); +BPF_HASH_MAP(pid_mm, __u32, __u64, 10240); + +/* Rebind pid's address space; mm == 0 unbinds at exit. The mm_owner entry is only + * dropped while it still names pid: a live CLONE_VM sibling shares the mm and must + * keep its registration. */ +static __always_inline void set_pid_mm(__u32 pid, __u64 mm) { + __u64* cur = bpf_map_lookup_elem(&pid_mm, &pid); + if (cur && *cur == mm) { + return; + } + if (cur) { + __u32* owner = bpf_map_lookup_elem(&mm_owner, cur); + if (owner && *owner == pid) { + bpf_map_delete_elem(&mm_owner, cur); + } + } + if (mm) { + bpf_map_update_elem(&pid_mm, &pid, &mm, BPF_ANY); + } else { + bpf_map_delete_elem(&pid_mm, &pid); + } +} + +/* Claim mm for pid without stealing from a live owner: CLONE_VM siblings share the + * mm, and overwriting would let the child's exec-time cleanup delete the entry out + * from under the still-live parent. */ +static __always_inline void mm_owner_take(__u64 mm, __u32 pid) { + __u32* reg = bpf_map_lookup_elem(&mm_owner, &mm); + if (!reg) { + bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); + return; + } + if (*reg == pid) { + return; + } + __u64* reg_mm = bpf_map_lookup_elem(&pid_mm, reg); + if (!reg_mm || *reg_mm != mm) { + bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); + } +} + +#define FOLIO_MAPPING_ANON 0x1UL + +const volatile __u32 page_shift = 12; + static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { e->data.rss.member = member; @@ -45,8 +103,13 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return 0; } owner = cur; - struct rss_owner state = {.pid = cur, .size = size}; + /* curr == 1 means current->mm is the counter's mm. pid_mm is maintained + * here too because the rmap hooks may not be attached. */ + struct task_struct* task = bpf_get_current_task_btf(); + __u64 mm = (__u64)BPF_CORE_READ(task, mm); + struct rss_owner state = {.pid = cur, .mm = mm, .size = size}; bpf_map_update_elem(&rss_counter_owner, &key, &state, BPF_ANY); + set_pid_mm(cur, mm); } else { struct rss_owner* found = bpf_map_lookup_elem(&rss_counter_owner, &key); if (!found) { @@ -59,6 +122,11 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { if (cur == owner) { return 0; } + + __u64* owner_mm = bpf_map_lookup_elem(&pid_mm, &owner); + if (!owner_mm || *owner_mm != found->mm) { + return 0; + } /* An external actor may only lower a counter. A larger value is a stale * reclaim read or an mm_id hash collision with another task; dropping it * keeps the reconstructed peak identical to the in-context timeline. */ @@ -71,6 +139,174 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return submit_rss_event(owner, ctx->member, size); } +/* Kernels < 6.18 store folio->flags as a bare unsigned long instead of + * memdesc_flags_t; probe which layout the running kernel has. */ +struct folio___legacy { + unsigned long flags; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_read_flags(struct folio* folio) { + if (bpf_core_field_exists(folio->flags.f)) { + return BPF_CORE_READ(folio, flags).f; + } + return BPF_CORE_READ((struct folio___legacy*)folio, flags); +} + +/* Kernels < 6.6 store the large-folio order in a dedicated byte instead of + * the low byte of _flags_1. */ +struct folio___order_byte { + unsigned char _folio_order; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_order(struct folio* folio) { + if (bpf_core_field_exists(((struct folio___order_byte*)folio)->_folio_order)) { + return BPF_CORE_READ((struct folio___order_byte*)folio, _folio_order); + } + return BPF_CORE_READ(folio, _flags_1) & 0xff; +} + +static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { + unsigned long flags = folio_read_flags(folio); + if (!(flags & (1UL << bpf_core_enum_value(enum pageflags, PG_head)))) { + return 1; + } + unsigned long order = folio_order(folio); + return 1UL << order; +} + +static __always_inline int folio_is_anon(struct folio* folio) { + unsigned long mapping = (unsigned long)BPF_CORE_READ(folio, mapping); + return (mapping & FOLIO_MAPPING_ANON) != 0; +} + +/* Mirrors the kernel's mm_counter(): anon folios are also swapbacked, so the + * anon check must come first. */ +static __always_inline __s32 folio_mm_counter(struct folio* folio) { + if (folio_is_anon(folio)) { + return MM_ANONPAGES; + } + unsigned long flags = folio_read_flags(folio); + if (flags & (1UL << bpf_core_enum_value(enum pageflags, PG_swapbacked))) { + return MM_SHMEMPAGES; + } + return MM_FILEPAGES; +} + +static __always_inline __u64 folio_page_address(struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + __u64 page_idx = ((__u64)page - (__u64)folio) / bpf_core_type_size(struct page); + __u64 pgoff = BPF_CORE_READ(folio, index) + page_idx; + __u64 vm_pgoff = BPF_CORE_READ(vma, vm_pgoff); + __u64 vm_start = BPF_CORE_READ(vma, vm_start); + if (pgoff < vm_pgoff) { + return vm_start; + } + return vm_start + ((pgoff - vm_pgoff) << page_shift); +} + +static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __s64 delta, + __u64 addr) { + __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); + struct task_struct* task = bpf_get_current_task_btf(); + __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 owner; + + if ((__u64)BPF_CORE_READ(task, mm) == mm) { + if (!is_tracked(pid)) { + return 0; + } + + mm_owner_take(mm, pid); + set_pid_mm(pid, mm); + owner = pid; + } else { + /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): + * recover the owner from the in-context registration. Fail toward dropping + * the event on any uncertainty about ownership. */ + __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); + if (!found) { + return 0; + } + owner = *found; + if (!is_tracked(owner)) { + return 0; + } + /* An mm_struct address may be reused while a stale owner entry remains. + * Accept only the current inverse binding. */ + __u64* owner_mm = bpf_map_lookup_elem(&pid_mm, &owner); + if (!owner_mm || *owner_mm != mm) { + return 0; + } + } + + /* header.tid is stamped from the current task; for a foreign actor it + * identifies the performer, not the owning pid. */ + SUBMIT_EVENT_AS(owner, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); +} + +SEC("fentry/folio_add_new_anon_rmap") +int BPF_PROG(fentry_folio_add_new_anon_rmap, struct folio* folio, struct vm_area_struct* vma, + unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_anon_rmap_ptes") +int BPF_PROG(fentry_folio_add_anon_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)nr_pages, address); +} + +SEC("fentry/folio_add_anon_rmap_pmd") +int BPF_PROG(fentry_folio_add_anon_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +static __always_inline int submit_file_rmap(struct folio* folio, struct page* page, + struct vm_area_struct* vma, __s64 delta) { + return submit_rmap(vma, folio_mm_counter(folio), delta, folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_ptes") +int BPF_PROG(fentry_folio_add_file_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, (__s64)nr_pages); +} + +SEC("fentry/folio_add_file_rmap_pmd") +int BPF_PROG(fentry_folio_add_file_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, (__s64)folio_nr_pages_est(folio)); +} + +SEC("fentry/folio_add_file_rmap_pud") +int BPF_PROG(fentry_folio_add_file_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, (__s64)folio_nr_pages_est(folio)); +} + +SEC("fentry/folio_remove_rmap_ptes") +int BPF_PROG(fentry_folio_remove_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, -(__s64)nr_pages); +} + +SEC("fentry/folio_remove_rmap_pmd") +int BPF_PROG(fentry_folio_remove_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, -(__s64)folio_nr_pages_est(folio)); +} + +SEC("fentry/folio_remove_rmap_pud") +int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_file_rmap(folio, page, vma, -(__s64)folio_nr_pages_est(folio)); +} + /* == Process lifecycle events == * * FORK lets userland seed a child's RSS from its parent at fork time: the @@ -113,6 +349,14 @@ int tracepoint_sched_process_exec(void* ctx) { return 0; } + /* SUBMIT_EVENT_AS returns, so the rebind must precede it. */ + struct task_struct* task = bpf_get_current_task_btf(); + __u64 new_mm = (__u64)BPF_CORE_READ(task, mm); + if (new_mm) { + mm_owner_take(new_mm, pid); + } + set_pid_mm(pid, new_mm); + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXEC, {}); } @@ -142,6 +386,10 @@ int tracepoint_sched_process_exit(void* ctx) { } bpf_map_delete_elem(&pids_ppid, &pid); + /* Drop the ownership mapping so foreign actors stop attributing to a pid + * the kernel may reuse. */ + set_pid_mm(pid, 0); + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXIT, {}); } diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index ef5d59b73..4ed422a58 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -89,6 +89,13 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.rss.size, }, ), + EVENT_TYPE_RMAP => ( + event.data.rmap.addr, + MemtrackEventKind::Rmap { + member: event.data.rmap.member, + delta: event.data.rmap.delta, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -222,6 +229,32 @@ mod tests { } } + #[test] + fn test_parse_rmap_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RMAP as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rmap.member = 3; + event.data.rmap.delta = 8; + event.data.rmap.addr = 0x7f00; + + let bytes = event_bytes(&event); + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0x7f00); + + match parsed.kind { + MemtrackEventKind::Rmap { member, delta } => { + assert_eq!(member, 3); + assert_eq!(delta, 8); + } + _ => panic!("Expected Rmap event kind"), + } + } + #[test] fn test_parse_fork_event() { let mut event: bindings::event = unsafe { std::mem::zeroed() }; diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index d7b7847cf..e7c5edf7e 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -145,3 +145,27 @@ macro_rules! attach_tracepoint { } }; } + +/// Invokes `$cb!` with each folio rmap kernel function base name; callbacks +/// build the `fentry_` program/method idents from it with `paste!`. +/// The PUD pair lives in its own group: `folio_remove_rmap_pud` only exists +/// since v6.15 (and both depend on arch PUD-THP support), so it is gated +/// separately from the core set present on every >= v6.8 kernel. +macro_rules! for_each_rmap_core_prog { + ($cb:ident) => { + $cb!(folio_add_new_anon_rmap); + $cb!(folio_add_anon_rmap_ptes); + $cb!(folio_add_anon_rmap_pmd); + $cb!(folio_add_file_rmap_ptes); + $cb!(folio_add_file_rmap_pmd); + $cb!(folio_remove_rmap_ptes); + $cb!(folio_remove_rmap_pmd); + }; +} + +macro_rules! for_each_rmap_pud_prog { + ($cb:ident) => { + $cb!(folio_add_file_rmap_pud); + $cb!(folio_remove_rmap_pud); + }; +} diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index f0b80b797..4bfc03ede 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,8 +1,9 @@ use crate::prelude::*; -use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; +use libbpf_rs::{AsRawLibbpf, Link}; use std::collections::HashMap; +use std::ffi::CString; use std::mem::MaybeUninit; use std::path::Path; @@ -94,6 +95,79 @@ fn symbol_file_offset<'a>( Some((address - section.address() + sh_offset) as usize) } +fn page_shift() -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "Failed to read system page size"); + Ok((page_size as u32).trailing_zeros()) +} + +fn kernel_func_exists(btf: &libbpf_rs::btf::Btf, name: &str) -> bool { + let Ok(name) = CString::new(name) else { + return false; + }; + + let id = unsafe { + libbpf_rs::libbpf_sys::btf__find_by_name_kind( + btf.as_libbpf_object().as_ptr(), + name.as_ptr(), + libbpf_rs::libbpf_sys::BTF_KIND_FUNC, + ) + }; + id >= 0 +} + +/// Which rmap fentry groups the kernel's BTF can support. The hooks are paired +/// add/remove accounting: attaching only a subset records adds without removes, +/// so reconstructed RSS grows monotonically. A missing core target therefore +/// disables the whole set, while the PUD pair (absent before v6.15 and on +/// arches without PUD THP) is dropped together without losing the core. +struct RmapSupport { + core: bool, + pud: bool, +} + +fn rmap_btf_support() -> Result { + let btf = libbpf_rs::btf::Btf::from_vmlinux().context("Failed to load vmlinux BTF")?; + let mut missing_core = Vec::new(); + let mut missing_pud = Vec::new(); + macro_rules! check_core_target { + ($name:ident) => { + if !kernel_func_exists(&btf, stringify!($name)) { + missing_core.push(stringify!($name)); + } + }; + } + macro_rules! check_pud_target { + ($name:ident) => { + if !kernel_func_exists(&btf, stringify!($name)) { + missing_pud.push(stringify!($name)); + } + }; + } + for_each_rmap_core_prog!(check_core_target); + for_each_rmap_pud_prog!(check_pud_target); + if !missing_core.is_empty() { + warn!( + "Kernel BTF is missing {}; disabling all rmap fentries", + missing_core.join(", ") + ); + return Ok(RmapSupport { + core: false, + pud: false, + }); + } + if !missing_pud.is_empty() { + warn!( + "Kernel BTF is missing {}; disabling PUD rmap fentries", + missing_pud.join(", ") + ); + } + Ok(RmapSupport { + core: true, + pud: missing_pud.is_empty(), + }) +} + /// Attach targets resolved from a library's symbol tables. pub struct ResolvedSymbols { offsets: HashMap, @@ -108,38 +182,77 @@ impl ResolvedSymbols { pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, + track_rmap: bool, + track_rmap_pud: bool, } impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new() -> Result { + pub fn new_with_rmap(track_rmap: bool) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant) + Self::with_variant(variant, track_rmap) } - /// Load a specific variant rather than the one [`Self::new`] would detect. - /// Either attaches given host privileges; the token only matters when - /// `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant) -> Result { - // Both variants expose `rodata_data` under the same field names, but as - // distinct generated types, so this can't be a function over the two. + /// Load a specific variant rather than the one [`Self::new_with_rmap`] + /// would detect. Either attaches given host privileges; the token only + /// matters when `bpf()` is called from an unprivileged user namespace. + pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + let page_shift = page_shift()?; + // Only the rmap targets are probed against BTF, so skip the vmlinux + // load entirely when they are not going to be attached. + let rmap = if track_rmap { + rmap_btf_support()? + } else { + RmapSupport { + core: false, + pud: false, + } + }; + + // Both variants expose `rodata_data` and `progs` under the same field + // names, but as distinct generated types, so this can't be a function + // over the two. macro_rules! open_and_load { ($builder:expr, $skel:path) => {{ let open_object = Box::leak(Box::new(MaybeUninit::uninit())); let mut open_skel = $builder .open(open_object) .context("Failed to open memtrack BPF skeleton")?; - if let (Some((dev, ino)), Some(rodata)) = ( - current_pidns_ids(), - open_skel.maps.rodata_data.as_deref_mut(), - ) { - rodata.target_pidns_dev = dev; - rodata.target_pidns_ino = ino; + + { + let rodata = open_skel + .maps + .rodata_data + .as_deref_mut() + .context("rodata map missing")?; + rodata.page_shift = page_shift; + if let Some((dev, ino)) = current_pidns_ids() { + rodata.target_pidns_dev = dev; + rodata.target_pidns_ino = ino; + } + } + + // Autoload is decided before load(), so fentries whose targets + // the kernel lacks have to be turned off here or the whole + // skeleton fails to load. + macro_rules! disable_rmap_prog { + ($name:ident) => { + paste::paste! { + open_skel.progs.[].set_autoload(false); + } + }; } + if !rmap.core { + for_each_rmap_core_prog!(disable_rmap_prog); + } + if !rmap.pud { + for_each_rmap_pud_prog!(disable_rmap_prog); + } + $skel(Box::new( open_skel .load() @@ -160,6 +273,8 @@ impl MemtrackBpf { Ok(Self { skel, probes: Vec::new(), + track_rmap: rmap.core, + track_rmap_pud: rmap.pud, }) } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index eec50638e..2aa2aede6 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -15,6 +15,27 @@ impl MemtrackBpf { if let Err(e) = self.attach_rss_stat() { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } + if self.track_rmap { + macro_rules! attach_rmap_prog { + ($name:ident) => { + paste! { + let link = with_skel!( + mut self, + skel => skel.progs.[].attach() + ) + .context(format!( + "Failed to attach {} fentry", + stringify!([]) + ))?; + self.probes.push(link); + } + }; + } + for_each_rmap_core_prog!(attach_rmap_prog); + if self.track_rmap_pud { + for_each_rmap_pud_prog!(attach_rmap_prog); + } + } Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 732fe50a5..075afd7bf 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -18,13 +18,19 @@ impl Tracker { /// Create a new tracker. The exec-mapping watcher discovers and attaches /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { - Self::with_bpf(MemtrackBpf::new()?) + let track_rmap = Self::track_rmap_from_env(); + Self::with_bpf(MemtrackBpf::new_with_rmap(track_rmap)?) } /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { - Self::with_bpf(MemtrackBpf::with_variant(variant)?) + let track_rmap = Self::track_rmap_from_env(); + Self::with_bpf(MemtrackBpf::with_variant(variant, track_rmap)?) + } + + fn track_rmap_from_env() -> bool { + std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1") } fn with_bpf(mut bpf: MemtrackBpf) -> Result { @@ -75,8 +81,8 @@ impl Tracker { } /// Enable allocator-event tracking in the BPF program. Lifetime events - /// (rss_stat, fork/exec/exit) are emitted for tracked pids regardless of - /// this toggle. + /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids + /// regardless of this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 17b4f1729..a6c610e8e 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..9) { + let kind = match rng.gen_range(0..10) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -29,6 +29,10 @@ fn generate_events(n: usize) -> Vec { member: rng.gen_range(0..4), size, }, + 9 => MemtrackEventKind::Rmap { + member: rng.gen_range(0..4), + delta: rng.gen_range(-1024..1024), + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 9fe78eb5f..b082a7c67 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -87,6 +87,10 @@ pub enum MemtrackEventKind { member: i32, size: u64, }, + Rmap { + member: i32, + delta: i64, + }, } pub struct MemtrackEventStream { From 8378a10dae4421fb25d682108e82b274ee615163 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 16:32:49 +0200 Subject: [PATCH 5/9] test(memtrack): cover RSS, rmap, and lifecycle event reconstruction Add the rss_tests integration suite: per-workload RSS/rmap reconstruction snapshots against /proc ground truth, fork-seeded child RSS, exec/exit resets, foreign-actor rmap attribution (reclaim, external madvise), and mm-ownership across CLONE_VM and exec. Extend tests/shared.rs with the tracker/fixture helpers these tests need and move compile_c_source into it for reuse. The suite needs two surfaces the production paths don't: a tracker mode that skips the allocator probes and exec-mapping watcher, and readers for the mm-ownership maps. --- crates/memtrack/Cargo.toml | 2 +- crates/memtrack/src/ebpf/memtrack/maps.rs | 44 + crates/memtrack/src/ebpf/memtrack/mod.rs | 2 + crates/memtrack/src/ebpf/mod.rs | 4 +- crates/memtrack/src/ebpf/tracker.rs | 64 +- crates/memtrack/testdata/rss/anon.c | 17 + crates/memtrack/testdata/rss/file.c | 33 + crates/memtrack/testdata/rss/file_region.h | 59 ++ crates/memtrack/testdata/rss/fork.c | 39 + crates/memtrack/testdata/rss/fork_idle.c | 49 ++ crates/memtrack/testdata/rss/madvise.c | 64 ++ crates/memtrack/testdata/rss/madvise_extern.c | 19 + crates/memtrack/testdata/rss/mremap_move.c | 31 + crates/memtrack/testdata/rss/munmap_hole.c | 40 + .../memtrack/testdata/rss/rmap_thread_fork.c | 58 ++ crates/memtrack/testdata/rss/rss_report.h | 58 ++ crates/memtrack/testdata/rss/shmem.c | 22 + crates/memtrack/testdata/rss/stale_mm_owner.c | 80 ++ crates/memtrack/testdata/rss/triangle.c | 24 + crates/memtrack/testdata/rss/vfork_spawn.c | 73 ++ crates/memtrack/tests/c_tests.rs | 28 +- crates/memtrack/tests/rss_tests.rs | 751 ++++++++++++++++++ crates/memtrack/tests/shared.rs | 133 +++- .../tests/snapshots/rss_tests__rss_anon.snap | 30 + .../tests/snapshots/rss_tests__rss_file.snap | 30 + .../tests/snapshots/rss_tests__rss_fork.snap | 42 + .../snapshots/rss_tests__rss_fork_idle.snap | 45 ++ .../snapshots/rss_tests__rss_madvise.snap | 30 + .../snapshots/rss_tests__rss_mremap_move.snap | 30 + .../snapshots/rss_tests__rss_munmap_hole.snap | 30 + .../tests/snapshots/rss_tests__rss_shmem.snap | 30 + .../snapshots/rss_tests__rss_triangle.snap | 30 + 32 files changed, 1934 insertions(+), 57 deletions(-) create mode 100644 crates/memtrack/testdata/rss/anon.c create mode 100644 crates/memtrack/testdata/rss/file.c create mode 100644 crates/memtrack/testdata/rss/file_region.h create mode 100644 crates/memtrack/testdata/rss/fork.c create mode 100644 crates/memtrack/testdata/rss/fork_idle.c create mode 100644 crates/memtrack/testdata/rss/madvise.c create mode 100644 crates/memtrack/testdata/rss/madvise_extern.c create mode 100644 crates/memtrack/testdata/rss/mremap_move.c create mode 100644 crates/memtrack/testdata/rss/munmap_hole.c create mode 100644 crates/memtrack/testdata/rss/rmap_thread_fork.c create mode 100644 crates/memtrack/testdata/rss/rss_report.h create mode 100644 crates/memtrack/testdata/rss/shmem.c create mode 100644 crates/memtrack/testdata/rss/stale_mm_owner.c create mode 100644 crates/memtrack/testdata/rss/triangle.c create mode 100644 crates/memtrack/testdata/rss/vfork_spawn.c create mode 100644 crates/memtrack/tests/rss_tests.rs create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_file.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 19b8382c6..d79e49eae 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -46,7 +46,7 @@ bindgen = "0.72" tempfile = { workspace = true } rstest = { workspace = true } test-log = { workspace = true } -insta = { workspace = true } +insta = { workspace = true, features = ["json", "redactions"] } test-with = { workspace = true } [package.metadata.dist] diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 42d2dd2e7..8e8ed8330 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -67,6 +67,50 @@ impl MemtrackBpf { "dropped_events", ) } + + pub fn ownership_maps(&self) -> Result { + let mm_owner = entries(with_skel!(self, skel => &skel.maps.mm_owner))?; + let pid_mm = entries(with_skel!(self, skel => &skel.maps.pid_mm))?; + Ok(OwnershipMaps { + mm_owner: mm_owner + .into_iter() + .map(|(mm, pid)| (mm, pid as u32)) + .collect(), + pid_mm: pid_mm + .into_iter() + .map(|(pid, mm)| (pid as u32, mm)) + .collect(), + }) + } +} + +/// Live ownership bindings: `mm_owner` (`mm_struct` pointer -> owning pid) and its +/// inverse `pid_mm`, which foreign-actor rmap attribution validates against. +pub struct OwnershipMaps { + pub mm_owner: Vec<(u64, u32)>, + pub pid_mm: Vec<(u32, u64)>, +} + +/// Iteration is `BPF_MAP_GET_NEXT_KEY` followed by a separate lookup, so it is not +/// atomic: a key deleted in between is skipped rather than reported. +fn entries(map: &impl MapCore) -> Result> { + let mut entries = Vec::new(); + for key in map.keys() { + if let Some(value) = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to read map entry")? + { + entries.push((le(&key), le(&value))); + } + } + Ok(entries) +} + +fn le(bytes: &[u8]) -> u64 { + bytes + .iter() + .rev() + .fold(0, |acc, &b| acc << 8 | u64::from(b)) } /// Read slot 0 of a single-entry `__u64` array map. diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 4bfc03ede..fcfb344d5 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -22,6 +22,8 @@ mod allocator; mod maps; mod tracking; +pub use maps::OwnershipMaps; + use crate::bpf_token::has_delegated_bpf_token; /// Which attach mechanism a loaded skeleton uses for its uprobes. See diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index cb6632e10..b77b299cf 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -6,5 +6,7 @@ mod proc_fs; mod spawn; mod tracker; -pub use memtrack::{BpfVariant, MemtrackBpf, ResolvedSymbols, resolve_symbol_offsets}; +pub use memtrack::{ + BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, resolve_symbol_offsets, +}; pub use tracker::Tracker; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 075afd7bf..914dfe22f 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,6 +1,6 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::{BpfVariant, MemtrackBpf}; +use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; @@ -12,6 +12,7 @@ use std::sync::mpsc; pub struct Tracker { bpf: Arc>, worker: Mutex>, + allocators: bool, } impl Tracker { @@ -19,39 +20,54 @@ impl Tracker { /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::with_bpf(MemtrackBpf::new_with_rmap(track_rmap)?) + Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, true) } /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::with_bpf(MemtrackBpf::with_variant(variant, track_rmap)?) + Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) } fn track_rmap_from_env() -> bool { std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1") } - fn with_bpf(mut bpf: MemtrackBpf) -> Result { + /// Track per-process RSS (rss_stat tracepoint and, when `track_rmap`, folio + /// rmap fentries) without allocator probes; no exec-mapping watcher or + /// attach worker runs. + pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { + Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, false) + } + + fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; - bpf.attach_exec_watcher()?; + if allocators { + bpf.attach_exec_watcher()?; + } let bpf = Arc::new(Mutex::new(bpf)); - let worker = AttachWorker::start(bpf.clone())?; + let worker = if allocators { + Some(AttachWorker::start(bpf.clone())?) + } else { + None + }; Ok(Self { bpf, - worker: Mutex::new(Some(worker)), + worker: Mutex::new(worker), + allocators, }) } /// Spawn `cmd` under tracking: the target is wrapped so it stops itself - /// before exec'ing, its pid is armed while stopped, then it is resumed. - /// The watcher observes the target's own `execve` mappings — no allocation - /// escapes untracked. + /// before exec'ing, its pid is armed while stopped, then it is resumed. When + /// the tracker runs an exec-mapping watcher, arming the pid before resume + /// ensures no allocation mapping escapes untracked; spawning after + /// [`Self::finish`] is an error in that mode. /// /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). @@ -63,11 +79,12 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - self.worker - .lock() - .as_ref() - .context("tracker already finished")? - .set_root_pid(pid); + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.allocators => bail!("tracker already finished"), + None => {} + } let (tx, rx) = mpsc::channel(); let poller = { @@ -98,15 +115,18 @@ impl Tracker { self.bpf.lock().dropped_events_count() } - /// Stop the attach worker and surface any fatal error it recorded, + /// Only meaningful while the BPF object is alive; teardown frees the maps. + pub fn ownership_maps(&self) -> Result { + self.bpf.lock().ownership_maps() + } + + /// Stop the attach worker, if any, and surface any fatal error it recorded, /// including missed exec mappings (incomplete allocator coverage). pub fn finish(&self) -> Result<()> { - let worker = self - .worker - .lock() - .take() - .context("tracker already finished")?; - worker.finish() + match self.worker.lock().take() { + Some(worker) => worker.finish(), + None => Ok(()), + } } /// Detach all attached probes. Called explicitly at teardown because the diff --git a/crates/memtrack/testdata/rss/anon.c b/crates/memtrack/testdata/rss/anon.c new file mode 100644 index 000000000..8aa0974ea --- /dev/null +++ b/crates/memtrack/testdata/rss/anon.c @@ -0,0 +1,17 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/file.c b/crates/memtrack/testdata/rss/file.c new file mode 100644 index 000000000..ac41168cd --- /dev/null +++ b/crates/memtrack/testdata/rss/file.c @@ -0,0 +1,33 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + /* Keep the data file next to the report: /tmp may be tmpfs (Ubuntu >= 25.04), + which accounts mapped file pages as shmem instead of file. */ + char template[4096]; + snprintf(template, sizeof(template), "%s.data-XXXXXX", argv[1]); + int fd = mkstemp(template); + if (fd < 0) return 1; + unlink(template); + char chunk[65536]; + memset(chunk, 0x42, sizeof(chunk)); + for (size_t off = 0; off < len; off += sizeof(chunk)) { + if (write(fd, chunk, sizeof(chunk)) != (ssize_t)sizeof(chunk)) return 1; + } + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret + (sink & 0); +} diff --git a/crates/memtrack/testdata/rss/file_region.h b/crates/memtrack/testdata/rss/file_region.h new file mode 100644 index 000000000..1877f114a --- /dev/null +++ b/crates/memtrack/testdata/rss/file_region.h @@ -0,0 +1,59 @@ +#ifndef FILE_REGION_H +#define FILE_REGION_H + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Maps len bytes of a private file mapping and faults every page in, so the + * pages land in the caller's RSS (MM_FILEPAGES) in-context and seed ownership. + * Returns the mapping, or NULL. + * + * The data file is derived from base rather than placed in /tmp, which is tmpfs + * on Ubuntu >= 25.04 and would account mapped file pages as shmem, not file. */ +static void* map_and_fault_file(const char* base, size_t len) { + char path[4096]; + snprintf(path, sizeof(path), "%s.data-XXXXXX", base); + int fd = mkstemp(path); + if (fd < 0) return NULL; + unlink(path); + if (ftruncate(fd, len) != 0) { + close(fd); + return NULL; + } + + /* The mapping keeps the inode alive, so the descriptor is not needed past mmap. */ + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (mem == MAP_FAILED) return NULL; + + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + (void)sink; + return mem; +} + +/* Pages out [mem, mem+len) of the calling process from a forked child, and waits + * for it. The reclaim must run in another task's context to exercise foreign-actor + * attribution. Returns 0 on success. */ +static int reclaim_from_child(void* mem, size_t len) { + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + int pidfd = syscall(SYS_pidfd_open, getppid(), 0); + if (pidfd < 0) _exit(1); + struct iovec iov = {.iov_base = mem, .iov_len = len}; + if (syscall(SYS_process_madvise, pidfd, &iov, 1UL, MADV_PAGEOUT, 0UL) < 0) _exit(2); + _exit(0); + } + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + return !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} + +#endif diff --git a/crates/memtrack/testdata/rss/fork.c b/crates/memtrack/testdata/rss/fork.c new file mode 100644 index 000000000..f6fcfac06 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +static int append_rss(const char* path, const char* label) { + long kb = rss_status_kb("RssAnon:"); + if (kb < 0) return 1; + FILE* report = fopen(path, "a"); + if (!report) return 1; + fprintf(report, "%s: %ld\n", label, kb); + fclose(report); + return 0; +} + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* parent = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (parent == MAP_FAILED) return 1; + memset(parent, 0x42, len); + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + void* child = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (child == MAP_FAILED) _exit(1); + memset(child, 0x24, len); + _exit(append_rss(argv[1], "ChildRssAnonKb")); + } + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + int ret = append_rss(argv[1], "ParentRssAnonKb"); + munmap(parent, len); + return ret || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/fork_idle.c b/crates/memtrack/testdata/rss/fork_idle.c new file mode 100644 index 000000000..0c5e56439 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork_idle.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The child only touches memory inherited from the parent: COW faults of anon + * pages are counter-neutral, so the child never reports its own RSS. Its + * footprint is only observable through fork-event seeding. The child must not + * allocate (no stdio/malloc), so the parent samples /proc//status while + * the child blocks on a pipe. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + + int ready[2]; + int release[2]; + if (pipe(ready) || pipe(release)) return 1; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + memset(mem, 0x24, len); + char b = 1; + if (write(ready[1], &b, 1) != 1) _exit(1); + if (read(release[0], &b, 1) != 1) _exit(1); + _exit(0); + } + + char b; + if (read(ready[0], &b, 1) != 1) return 1; + long child_anon = rss_status_kb_pid(pid, "RssAnon:"); + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ChildRssAnon: %ld\n", child_anon); + fclose(report); + if (write(release[1], &b, 1) != 1) return 1; + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + return ret || child_anon < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/madvise.c b/crates/memtrack/testdata/rss/madvise.c new file mode 100644 index 000000000..e5f779725 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* AnonHugePages for the whole process, from smaps_rollup (not in /proc/status). */ +static long anon_huge_pages_kb(void) { + FILE* rollup = fopen("/proc/self/smaps_rollup", "r"); + if (!rollup) return 0; + char line[256]; + long kb = 0; + while (fgets(line, sizeof(line), rollup)) { + if (sscanf(line, "AnonHugePages: %ld", &kb) == 1) break; + } + fclose(rollup); + return kb; +} + +/* Two 32 MiB anon regions: one advised MADV_HUGEPAGE (2 MiB-aligned so the + * kernel *may* fault it as PMD folios), one MADV_NOHUGEPAGE (guaranteed pte + * path). MADV_HUGEPAGE is only advisory, so nothing here depends on THP + * actually materializing: the accounted totals are identical either way. + * + * Both regions are dropped with MADV_DONTNEED and faulted a second time. If + * the in-context removes were missed, the reconstructed running total reaches + * 128 MiB instead of 64 MiB, so the snapshot peak is the assertion. The report + * also carries the observed AnonHugePages (hex, so it stays out of the + * snapshot) letting the test require PMD-sized rmap deltas exactly when THP + * actually materialized. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + const size_t align = 2UL * 1024 * 1024; + + void* raw = mmap(NULL, len + align, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) return 1; + char* huge = (char*)(((uintptr_t)raw + align - 1) & ~(uintptr_t)(align - 1)); + if (madvise(huge, len, MADV_HUGEPAGE) != 0) return 1; + + char* base = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) return 1; + if (madvise(base, len, MADV_NOHUGEPAGE) != 0) return 1; + + memset(huge, 0x42, len); + memset(base, 0x42, len); + + if (madvise(huge, len, MADV_DONTNEED) != 0) return 1; + if (madvise(base, len, MADV_DONTNEED) != 0) return 1; + + memset(huge, 0x43, len); + memset(base, 0x43, len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ThpKb: 0x%lx\n", anon_huge_pages_kb()); + fclose(report); + munmap(raw, len + align); + munmap(base, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/madvise_extern.c b/crates/memtrack/testdata/rss/madvise_extern.c new file mode 100644 index 000000000..454566b6c --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise_extern.c @@ -0,0 +1,19 @@ +#define _GNU_SOURCE +#include + +#include "file_region.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); /* let the tracker attach + enable + add root pid */ + + size_t len = 64UL * 1024 * 1024; + void* mem = map_and_fault_file(argv[1], len); + if (!mem) return 1; + + if (reclaim_from_child(mem, len) != 0) return 1; + + sleep(1); /* let the external decrement flush to the ring buffer */ + /* No munmap: an in-context decrement would mask the external-path signal. */ + return 0; +} diff --git a/crates/memtrack/testdata/rss/mremap_move.c b/crates/memtrack/testdata/rss/mremap_move.c new file mode 100644 index 000000000..3b86fc039 --- /dev/null +++ b/crates/memtrack/testdata/rss/mremap_move.c @@ -0,0 +1,31 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +/* Fault 32 MiB, then force mremap to move it to a reserved destination. The + * move relocates page tables without rmap remove/add, so no events should + * fire and the second memset must not refault: a peak of 64 MiB instead of + * 32 MiB means the move was double-counted or the pages were dropped. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + + char* src = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (src == MAP_FAILED) return 1; + memset(src, 0x42, len); + + void* reserved = mmap(NULL, len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) return 1; + + char* dst = mremap(src, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, reserved); + if (dst == MAP_FAILED) return 1; + memset(dst, 0x43, len); + + int ret = write_rss_report(argv[1]); + munmap(dst, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/munmap_hole.c b/crates/memtrack/testdata/rss/munmap_hole.c new file mode 100644 index 000000000..e769ff5b7 --- /dev/null +++ b/crates/memtrack/testdata/rss/munmap_hole.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Fault 64 MiB, punch a 16 MiB hole with munmap, then map and fault the hole + * again. If the partial-unmap removes were missed (or covered the wrong + * range), the reconstructed running total peaks at 80 MiB instead of 64 MiB. + * MADV_NOHUGEPAGE keeps every folio a single page: the region is only + * page-aligned, so a straddling PMD folio would blur the hole boundaries. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 64UL * 1024 * 1024; + const size_t hole_off = 24UL * 1024 * 1024; + const size_t hole_len = 16UL * 1024 * 1024; + + char* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + if (madvise(mem, len, MADV_NOHUGEPAGE) != 0) return 1; + memset(mem, 0x42, len); + + if (munmap(mem + hole_off, hole_len) != 0) return 1; + + void* refill = mmap(mem + hole_off, hole_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (refill == MAP_FAILED) return 1; + if (madvise(refill, hole_len, MADV_NOHUGEPAGE) != 0) return 1; + memset(refill, 0x43, hole_len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "Layout: 0x%lx 0x%lx 0x%lx 0x%lx\n", (unsigned long)mem, hole_off, hole_len, + len); + fclose(report); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rmap_thread_fork.c b/crates/memtrack/testdata/rss/rmap_thread_fork.c new file mode 100644 index 000000000..f7ca7cef4 --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_thread_fork.c @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* A worker thread (not the group leader) calls fork(); the child faults an + * anon region and writes the report. Child tracking must key on the parent's + * TGID: a scheme keyed on the raw creator tid would only cover forks issued + * by the leader. */ + +static const char* report_path; + +static void* worker(void* arg) { + (void)arg; + + pid_t pid = fork(); + if (pid < 0) { + _exit(1); + } + if (pid == 0) { + size_t region = 64UL * 1024 * 1024; + void* mem = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(mem, region); + _exit(ret); + } + + int status; + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + _exit(1); + } + return NULL; +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + if (pthread_join(thread, NULL) != 0) { + return 1; + } + return 0; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h new file mode 100644 index 000000000..169bdc077 --- /dev/null +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -0,0 +1,58 @@ +#ifndef RSS_REPORT_H +#define RSS_REPORT_H + +#include +#include +#include + +static long rss_status_kb_pid(int pid, const char* key) { + char status_path[64]; + snprintf(status_path, sizeof(status_path), "/proc/%d/status", pid); + FILE* status = fopen(status_path, "r"); + if (!status) return -1; + char line[256]; + long kb = -1; + size_t key_len = strlen(key); + while (fgets(line, sizeof(line), status)) { + if (strncmp(line, key, key_len) == 0 && sscanf(line + key_len, " %ld", &kb) == 1) { + break; + } + } + fclose(status); + return kb; +} + +static long rss_status_kb(const char* key) { + return rss_status_kb_pid(getpid(), key); +} + +/* Reads Rss* through /proc//status of the given task. For the + * thread-group dir this is the leader's task: once the leader is a zombie its + * mm pointer is gone and the Rss lines disappear, so callers whose leader has + * exited must pass a live thread's tid instead. */ +static int write_rss_report_pid(int pid, const char* path) { + long anon = rss_status_kb_pid(pid, "RssAnon:"); + long file = rss_status_kb_pid(pid, "RssFile:"); + long shmem = rss_status_kb_pid(pid, "RssShmem:"); + /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which + * survives execve and so reports the peak of the pre-exec parent image. + * VmHWM belongs to the mm and starts fresh at exec. */ + long max_rss = rss_status_kb_pid(pid, "VmHWM:"); + if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { + return 1; + } + FILE* report = fopen(path, "w"); + if (!report) { + return 1; + } + fprintf(report, "RssAnon: %ld\nRssFile: %ld\nRssShmem: %ld\nMaxRssKb: %ld\n", anon, file, + shmem, max_rss); + fclose(report); + return 0; +} + +static int write_rss_report(const char* path) { + return write_rss_report_pid(getpid(), path); +} + +#endif diff --git a/crates/memtrack/testdata/rss/shmem.c b/crates/memtrack/testdata/rss/shmem.c new file mode 100644 index 000000000..63b53e013 --- /dev/null +++ b/crates/memtrack/testdata/rss/shmem.c @@ -0,0 +1,22 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + int fd = memfd_create("memtrack-rss-shmem", 0); + if (fd < 0) return 1; + if (ftruncate(fd, len) != 0) return 1; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret; +} diff --git a/crates/memtrack/testdata/rss/stale_mm_owner.c b/crates/memtrack/testdata/rss/stale_mm_owner.c new file mode 100644 index 000000000..19e0e25c8 --- /dev/null +++ b/crates/memtrack/testdata/rss/stale_mm_owner.c @@ -0,0 +1,80 @@ +/* Recycled mm_struct, stale rss_stat owner. + * + * A child faults a page (registering mm0's hash as owned by that pid), execs into + * "big" (freeing mm0 while the pid lives on), and idles holding REGION_MIB of anon + * RSS on mm1. Each churn fork then allocates an mm_struct from the same slab and + * populates it from the PARENT's context before tearing it down from the child's, + * so both updates are out-of-context. mm0's slot sits at the head of the per-cpu + * freelist, so the whole burst hashes to mm0's id. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "rss_report.h" + +#define REGION_MIB 256 +#define CHURN_FORKS 256 + +/* Keeps the mm_struct freed at exec at the head of this CPU's slab freelist, which + * is LIFO only per cpu. */ +static int pin_one_cpu(void) { + cpu_set_t one; + CPU_ZERO(&one); + CPU_SET(sched_getcpu(), &one); + return sched_setaffinity(0, sizeof(one), &one) != 0; +} + +static int run_big(const char* report_path) { + size_t len = (size_t)REGION_MIB * 1024 * 1024; + void* region = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (region == MAP_FAILED) return 1; + memset(region, 0x42, len); + if (write_rss_report(report_path) != 0) return 1; + /* Idle while the parent churns: no further fault means no in-context + * rss_stat event repairs a bogus sample. */ + pause(); + return 0; +} + +int main(int argc, char** argv) { + if (argc < 2) return 1; + if (argc >= 3 && strcmp(argv[2], "big") == 0) return run_big(argv[1]); + + /* Inherited across both the fork and the exec below. */ + if (pin_one_cpu() != 0) return 1; + + pid_t big = fork(); + if (big < 0) return 1; + if (big == 0) { + void* page = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (page == MAP_FAILED) _exit(1); + memset(page, 1, 4096); + char* args[] = {argv[0], argv[1], "big", NULL}; + execv("/proc/self/exe", args); + _exit(1); + } + + /* run_big writes the report right after its memset, so its existence + * means the region is resident. */ + for (int i = 0; access(argv[1], F_OK) != 0; i++) { + if (i > 5000) return 1; + usleep(1000); + } + + for (int i = 0; i < CHURN_FORKS; i++) { + pid_t churn = fork(); + if (churn < 0) return 1; + if (churn == 0) _exit(0); + int churn_status; + if (waitpid(churn, &churn_status, 0) < 0) return 1; + } + + if (kill(big, SIGKILL) != 0) return 1; + int status; + return waitpid(big, &status, 0) < 0; +} diff --git a/crates/memtrack/testdata/rss/triangle.c b/crates/memtrack/testdata/rss/triangle.c new file mode 100644 index 000000000..69bf7bf64 --- /dev/null +++ b/crates/memtrack/testdata/rss/triangle.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t chunk = 16UL * 1024 * 1024; + void* bufs[4]; + for (int i = 0; i < 4; i++) { + bufs[i] = mmap(NULL, chunk, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bufs[i] == MAP_FAILED) return 1; + memset(bufs[i], 0x42, chunk); + usleep(50 * 1000); + } + int ret = write_rss_report(argv[1]); + for (int i = 0; i < 4; i++) { + munmap(bufs[i], chunk); + usleep(50 * 1000); + } + return ret; +} diff --git a/crates/memtrack/testdata/rss/vfork_spawn.c b/crates/memtrack/testdata/rss/vfork_spawn.c new file mode 100644 index 000000000..75b96723c --- /dev/null +++ b/crates/memtrack/testdata/rss/vfork_spawn.c @@ -0,0 +1,73 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include "file_region.h" + +/* Exercise ownership of an mm shared through CLONE_VM. + * + * The fixture pauses before later parent faults can refresh the binding, then + * reclaims the parent's file pages from another process. Ownership must remain + * with the parent throughout. + * + * argv: [1] = checkpoint-ready file to create, [2] = release file to wait for. */ + +#define REGION (64UL * 1024 * 1024) +#define SCRATCH (1UL * 1024 * 1024) +#define CHILD_STACK (256UL * 1024) + +static char* scratch; + +/* Fault fresh anonymous pages into the shared address space from the child. */ +static int clone_vm_child(void* arg) { + (void)arg; + memset(scratch, 0x42, SCRATCH); + /* POSIX guarantees /bin/sh; only the exec transition matters. */ + execl("/bin/sh", "sh", "-c", "exit 0", (char*)NULL); + _exit(127); +} + +int main(int argc, char** argv) { + if (argc < 3) return 2; + sleep(1); /* let the tracker attach + enable + add root pid */ + + void* mem = map_and_fault_file(argv[1], REGION); + if (!mem) return 1; + + /* Leave the mapping untouched so only the CLONE_VM child faults its pages. */ + scratch = mmap(NULL, SCRATCH, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) return 1; + + char* stack = mmap(NULL, CHILD_STACK, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) return 1; + + /* glibc's posix_spawn uses CLONE_VM|CLONE_VFORK. CLONE_VFORK orders the + * child's exec before the later foreign reclaim. */ + pid_t c = clone(clone_vm_child, stack + CHILD_STACK, CLONE_VM | CLONE_VFORK | SIGCHLD, NULL); + if (c < 0) return 1; + int status; + if (waitpid(c, &status, 0) < 0) return 1; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 1; + + /* Pause before a parent-side fault can refresh the ownership binding. Use + * only already-faulted memory until the test releases the fixture. */ + int rfd = creat(argv[1], 0644); + if (rfd < 0) return 1; + close(rfd); + /* Bounded so a dead test cannot leave an orphan here holding the mapping. */ + for (int i = 0; access(argv[2], F_OK) != 0; i++) { + if (i > 1500) return 3; + usleep(20000); + } + + if (reclaim_from_child(mem, REGION) != 0) return 1; + sleep(1); /* let the external decrements flush to the ring buffer */ + /* No munmap: an in-context decrement would mask the external-path signal. */ + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 17fb0a3ad..b67eaa9d7 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -2,35 +2,9 @@ mod shared; use rstest::rstest; -use std::fs; -use std::path::Path; use std::process::Command; use tempfile::TempDir; -/// Compiles C source code and returns the binary path -fn compile_c_source( - source_code: &str, - name: &str, - output_dir: &Path, -) -> Result> { - let source_path = output_dir.join(format!("{name}.c")); - let binary_path = output_dir.join(name); - - fs::write(&source_path, source_code)?; - - let output = Command::new("gcc") - .args(["-o", binary_path.to_str().unwrap()]) - .arg(&source_path) - .output()?; - - if !output.status.success() { - eprintln!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); - return Err("Failed to compile C fixture".into()); - } - - Ok(binary_path) -} - struct AllocationTestCase { name: &'static str, source: &'static str, @@ -96,7 +70,7 @@ fn test_allocation_tracking( #[case] test_case: &AllocationTestCase, ) -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = compile_c_source(test_case.source, test_case.name, temp_dir.path())?; + let binary = shared::compile_c_source(test_case.source, test_case.name, temp_dir.path())?; assert_events_snapshot_for_each_variant!(test_case.name, || Command::new(&binary))?; diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs new file mode 100644 index 000000000..847027f1d --- /dev/null +++ b/crates/memtrack/tests/rss_tests.rs @@ -0,0 +1,751 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tempfile::TempDir; + +const MIB: u64 = 1024 * 1024; + +/// Quantize to 16 MiB buckets so snapshots tolerate page-level noise between runs. +fn mib_16(bytes: u64) -> u64 { + (bytes + 8 * MIB) / (16 * MIB) * 16 +} + +fn page_size() -> u64 { + let size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + assert!(size > 0, "sysconf(_SC_PAGESIZE) failed"); + size as u64 +} + +fn parse_report(report: &str) -> BTreeMap { + report + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let key = parts.next()?.to_string(); + let kb: u64 = parts.next()?.parse().ok()?; + Some((key, mib_16(kb * 1024))) + }) + .collect() +} + +#[derive(Debug, Serialize)] +struct PidRss { + pid: i32, + file_mib: u64, + anon_mib: u64, + shmem_mib: u64, + max_rss_mib: u64, +} + +#[derive(Serialize)] +struct RssSummary { + report: BTreeMap, + rss_stat: Vec, + rmap: Vec, +} + +#[derive(Default)] +struct PeakAccum { + /// rss_stat/rmap member order: file, anon, swap, shmem. `max_rss` excludes + /// swap (index 2), which is not resident. + latest: [i64; 4], + peaks: [i64; 4], + max_rss: i64, +} + +impl PeakAccum { + /// Absolute assignment (rss_stat: the kernel counter's current value). + fn set(&mut self, index: usize, bytes: i64) { + self.latest[index] = bytes; + self.update_peaks(); + } + + /// Delta accumulation (rmap: summed folio add/remove deltas from zero). + fn add(&mut self, index: usize, delta_bytes: i64) { + self.latest[index] += delta_bytes; + self.update_peaks(); + } + + /// Fork: child inherits the parent's current resident values. + fn seed(&mut self, latest: [i64; 4]) { + self.latest = latest; + self.update_peaks(); + } + + /// Exec/Exit: reset the running value only; recorded peaks are retained + /// (an absolute rss_stat re-syncs on the next event; rmap re-accumulates). + fn reset(&mut self) { + self.latest = [0; 4]; + } + + fn update_peaks(&mut self) { + for (peak, latest) in self.peaks.iter_mut().zip(self.latest) { + *peak = (*peak).max(latest); + } + self.max_rss = self + .max_rss + .max(self.latest[0] + self.latest[1] + self.latest[3]); + } +} + +/// Reduce the raw event stream to per-pid resident peaks in BYTES. +/// Returns first-activity pid order plus rss_stat and rmap accumulators. +fn per_pid_raw( + events: &[MemtrackEvent], +) -> (Vec, BTreeMap, BTreeMap) { + // Pids can wrap, so numeric order is unstable; both views instead share one + // first-activity order, keeping rows aligned after pids are redacted. + let mut order: Vec = Vec::new(); + let mut rss: BTreeMap = BTreeMap::new(); + let mut rmap: BTreeMap = BTreeMap::new(); + + fn seen(order: &mut Vec, pid: i32) { + if !order.contains(&pid) { + order.push(pid); + } + } + + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + rss.entry(event.pid).or_default().set(index, size as i64); + } + MemtrackEventKind::Rmap { member, delta } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + rmap.entry(event.pid) + .or_default() + .add(index, delta * page_size() as i64); + } + MemtrackEventKind::Fork { parent_pid } => { + seen(&mut order, event.pid); + let seed = rss.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + rss.entry(event.pid).or_default().seed(seed); + let seed = rmap.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + rmap.entry(event.pid).or_default().seed(seed); + } + MemtrackEventKind::Exec | MemtrackEventKind::Exit => { + if let Some(acc) = rss.get_mut(&event.pid) { + acc.reset(); + } + if let Some(acc) = rmap.get_mut(&event.pid) { + acc.reset(); + } + } + _ => {} + } + } + (order, rss, rmap) +} + +fn per_pid_peaks(events: &[MemtrackEvent]) -> (Vec, Vec) { + let (order, rss, rmap) = per_pid_raw(events); + let project = |map: &BTreeMap| -> Vec { + order + .iter() + .filter_map(|pid| { + let acc = map.get(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0].max(0) as u64), + anon_mib: mib_16(acc.peaks[1].max(0) as u64), + shmem_mib: mib_16(acc.peaks[3].max(0) as u64), + max_rss_mib: mib_16(acc.max_rss.max(0) as u64), + }) + }) + .collect() + }; + (project(&rss), project(&rmap)) +} + +/// Headers the fixtures `#include`. Each fixture compiles in its own temp dir, +/// so they are copied in next to the source. +fn write_fixture_headers(dir: &Path) -> std::io::Result<()> { + for (name, source) in [ + ("rss_report.h", include_str!("../testdata/rss/rss_report.h")), + ( + "file_region.h", + include_str!("../testdata/rss/file_region.h"), + ), + ] { + std::fs::write(dir.join(name), source)?; + } + Ok(()) +} + +/// Compile a fixture and build the command that runs it against a report path +/// in the returned temp dir, which must outlive the run. +fn build_fixture( + source: &str, + name: &str, +) -> Result<(TempDir, PathBuf, Command), Box> { + // Fixtures mmap data files created next to the report path, so the temp dir + // must be disk-backed: /tmp may be tmpfs (Ubuntu >= 25.04), which accounts + // mapped file pages as shmem instead of file. + std::fs::create_dir_all(env!("CARGO_TARGET_TMPDIR"))?; + let temp_dir = TempDir::new_in(env!("CARGO_TARGET_TMPDIR"))?; + write_fixture_headers(temp_dir.path())?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let report_path = temp_dir.path().join(format!("{name}.report")); + let mut command = Command::new(&binary); + command.arg(&report_path); + Ok((temp_dir, report_path, command)) +} + +/// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to +/// its argv[1] alongside the collected events. +/// +/// The report read is best-effort: some fixtures write no report. +fn track_fixture( + source: &str, + name: &str, + track: impl FnOnce(Command) -> shared::TrackResult, +) -> Result<(Option, Vec), Box> { + let (_temp_dir, report_path, command) = build_fixture(source, name)?; + let (events, thread_handle) = track(command)?; + let raw_report = std::fs::read_to_string(&report_path).ok(); + thread_handle.join().unwrap(); + Ok((raw_report, events)) +} + +/// [`track_fixture`] returning the ownership maps snapshotted once the tracked +/// tree exited, instead of the report text. +fn track_fixture_with_maps( + source: &str, + name: &str, +) -> Result<(Vec, shared::OwnershipMaps), Box> { + let (_temp_dir, _report_path, command) = build_fixture(source, name)?; + let (events, maps, thread_handle) = shared::track_command_with_rmap_maps(command)?; + thread_handle.join().unwrap(); + Ok((events, maps)) +} + +/// Every fork as `(parent_pid, child_pid)`, in timestamp order. +fn fork_pairs(events: &[MemtrackEvent]) -> Vec<(i32, i32)> { + events + .iter() + .sorted_by_key(|e| e.timestamp) + .filter_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .collect() +} + +/// The first fork observed: `(parent_pid, child_pid)`. +fn first_fork_pair(events: &[MemtrackEvent]) -> Option<(i32, i32)> { + fork_pairs(events).first().copied() +} + +/// Summed rmap bytes for one pid's `member` counter, in one direction. `tid` +/// narrows to a single performing task, which is how a foreign actor's events +/// are told apart from the owner's own. +fn rmap_bytes( + events: &[MemtrackEvent], + pid: i32, + tid: Option, + member: i32, + positive: bool, +) -> u64 { + events + .iter() + .filter(|e| e.pid == pid && tid.is_none_or(|tid| e.tid == tid)) + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: m, delta } + if m == member && (delta > 0) == positive => + { + Some(delta.unsigned_abs()) + } + _ => None, + }) + .sum::() + * page_size() +} + +/// Pins reconstructed rmap addresses to the exact punched range: hole pages are +/// the only ones removed and later re-added; every other page in the region is +/// added first and only removed afterwards. +/// +/// Only own-context events (tid == pid; the fixture is single-threaded) are +/// considered: foreign actors like kcompactd migrating a page produce a +/// remove-then-add on arbitrary pages, which the attribution of foreign rmap +/// events makes visible here. +fn assert_rmap_hole_addresses( + events: &[MemtrackEvent], + base: u64, + hole_off: u64, + hole_len: u64, + len: u64, +) { + let page = page_size(); + let n_pages = (len / page) as usize; + let mut first_remove = vec![u64::MAX; n_pages]; + let mut added = vec![false; n_pages]; + let mut readded = vec![false; n_pages]; + + for event in events.iter().sorted_by_key(|e| e.timestamp) { + let MemtrackEventKind::Rmap { delta, .. } = event.kind else { + continue; + }; + if event.tid != event.pid { + continue; + } + if event.addr < base || event.addr >= base + len { + continue; + } + let first = ((event.addr - base) / page) as usize; + let last = (first + delta.unsigned_abs() as usize).min(n_pages); + for page in first..last { + if delta > 0 { + added[page] = true; + if event.timestamp > first_remove[page] { + readded[page] = true; + } + } else { + first_remove[page] = first_remove[page].min(event.timestamp); + } + } + } + + let hole = (hole_off / page) as usize..((hole_off + hole_len) / page) as usize; + for page in 0..n_pages { + assert!(added[page], "page {page} never saw an rmap add"); + assert_eq!( + readded[page], + hole.contains(&page), + "page {page}: remove-then-add pattern does not match the hole range" + ); + } +} + +/// Both accounting modes must reproduce a fixture's self-reported `/proc` RSS +/// peaks; fixtures with a Layout line additionally pin rmap events to exact +/// addresses, and fixtures with THP additionally check huge-folio deltas. +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::anon(include_str!("../testdata/rss/anon.c"), "anon")] +#[case::file(include_str!("../testdata/rss/file.c"), "file")] +#[case::shmem(include_str!("../testdata/rss/shmem.c"), "shmem")] +#[case::fork(include_str!("../testdata/rss/fork.c"), "fork")] +#[case::fork_idle(include_str!("../testdata/rss/fork_idle.c"), "fork_idle")] +#[case::triangle(include_str!("../testdata/rss/triangle.c"), "triangle")] +#[case::madvise(include_str!("../testdata/rss/madvise.c"), "madvise")] +#[case::munmap_hole(include_str!("../testdata/rss/munmap_hole.c"), "munmap_hole")] +#[case::mremap_move(include_str!("../testdata/rss/mremap_move.c"), "mremap_move")] +fn test_rss_rmap_tracking( + #[case] source: &str, + #[case] name: &str, +) -> Result<(), Box> { + let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let raw_report = raw_report.ok_or("fixture wrote no rss report")?; + let (rss_stat, rmap) = per_pid_peaks(&events); + let summary = RssSummary { + report: parse_report(&raw_report), + rss_stat, + rmap, + }; + insta::assert_json_snapshot!(format!("rss_{name}"), summary, { + ".rss_stat[].pid" => "[pid]", + ".rmap[].pid" => "[pid]", + }); + + if let Some(layout) = raw_report + .lines() + .find_map(|line| line.strip_prefix("Layout:")) + { + let values: Vec = layout + .split_whitespace() + .map(|token| u64::from_str_radix(token.trim_start_matches("0x"), 16)) + .collect::>()?; + let [base, hole_off, hole_len, len] = values[..] else { + panic!("malformed Layout line: {layout}"); + }; + assert_rmap_hole_addresses(&events, base, hole_off, hole_len, len); + } + + // ThpKb > 0 means the MADV_HUGEPAGE region really faulted as PMD folios, so + // huge-folio accounting must be visible: a +512-page delta from the new-anon + // fault path and a -512-page delta that can only come from the + // folio_remove_rmap_pmd hook (MADV_DONTNEED / munmap of a pmd-mapped THP). + if let Some(thp) = raw_report + .lines() + .find_map(|line| line.strip_prefix("ThpKb:")) + { + let thp_kb = u64::from_str_radix(thp.trim().trim_start_matches("0x"), 16)?; + if thp_kb > 0 { + let deltas = events.iter().filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { delta, .. } => Some(delta), + _ => None, + }); + let (mut huge_add, mut huge_remove) = (false, false); + for delta in deltas { + huge_add |= delta >= 512; + huge_remove |= delta <= -512; + } + assert!( + huge_add, + "THP present ({thp_kb} kB) but no huge-folio rmap add" + ); + assert!( + huge_remove, + "THP present ({thp_kb} kB) but no pmd-sized rmap remove" + ); + } + } + Ok(()) +} + +/// A foreign actor reclaiming another process's memory must be attributed to +/// the OWNER of that memory, not the actor. The fixture forks a child B that +/// calls process_madvise(MADV_PAGEOUT) against parent A's file region from B's +/// own context (tid == B). Both accounting modes must show A's in-context faults +/// AND the foreign reclaim charged back to A via the mm_owner map. +enum Reclaim { + /// Absolute file-RSS updates; a foreign reclaim appears as a decrement. + RssStat, + /// Reconstructed file-page deltas; a foreign reclaim appears as removes. + Rmap, +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::rss_stat(Reclaim::RssStat)] +#[case::rmap(Reclaim::Rmap)] +fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { + let track: fn(Command) -> shared::TrackResult = match mode { + Reclaim::RssStat => shared::track_command, + Reclaim::Rmap => shared::track_command_with_rmap, + }; + let (_report, events) = track_fixture( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + track, + )?; + + // A = owner that faulted the file region; B = external caller, single-threaded + // so its tid == its pid. + let (a, b) = first_fork_pair(&events).expect("expected a fork event"); + + match mode { + Reclaim::RssStat => { + let peak = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); + + // A file decrement owned by A but emitted from B's context: only present + // when out-of-context rss_stat updates are attributed to the owner. + let external_decrement = events.iter().any(|e| { + e.pid == a + && e.tid == b + && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) + }); + assert!( + external_decrement, + "external file-RSS decrement not attributed to A (tid=B)" + ); + } + Reclaim::Rmap => { + let in_context_bytes = rmap_bytes(&events, a, None, 0, true); + assert!( + in_context_bytes >= 32 * MIB, + "in-context file rmap adds too small: {in_context_bytes}" + ); + + // A file-page remove owned by A but emitted from B's context (tid == B), + // only present when the foreign reclaim's rmap events are attributed to + // the owner via the mm_owner map. + let external_bytes = rmap_bytes(&events, a, Some(b), 0, false); + assert!( + external_bytes >= 8 * MIB, + "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B): only {external_bytes} bytes" + ); + } + } + Ok(()) +} + +/// An rss_stat ownership registration is keyed on the kernel's `mm_id` hash, which +/// outlives the mm it was seeded from: once the owner execs, the freed `mm_struct` +/// can be recycled by an unrelated fork whose near-zero counters hash to the same +/// id. Those counters must be dropped, not emitted as the owner's absolute RSS. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_stale_mm_owner_keeps_live_rss() -> Result<(), Box> { + let (events, maps) = track_fixture_with_maps( + include_str!("../testdata/rss/stale_mm_owner.c"), + "stale_mm_owner", + )?; + + // The only fork before the burst is the child that execs into the memory-holding + // image, so it keeps its pid across the exec. + let (_parent, big) = first_fork_pair(&events).ok_or("no fork event")?; + let exec_ts = events + .iter() + .find(|e| e.pid == big && matches!(e.kind, MemtrackEventKind::Exec)) + .map(|e| e.timestamp) + .ok_or("no exec event for the memory-holding child")?; + + // The pre-exec image only faulted a single page; its mm is the one whose slab + // slot gets reused. + let anon_after_exec: Vec<_> = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 1, size } if e.pid == big && e.timestamp > exec_ts => { + Some((e, size)) + } + _ => None, + }) + .collect(); + + let (peak_ts, peak) = anon_after_exec + .iter() + .max_by_key(|&&(_, size)| size) + .map(|&(e, size)| (e.timestamp, size)) + .ok_or("no anon rss_stat event after the exec")?; + assert!(peak >= 128 * MIB, "peak anon RSS too small: {peak}"); + + // Past the peak the region is held untouched until the fixture is killed, so no + // legitimate absolute sample may fall back near zero. + let collapsed: Vec<_> = anon_after_exec + .iter() + .filter(|&&(e, size)| e.timestamp > peak_ts && size < peak / 4) + .map(|&(e, size)| (e.tid, size)) + .collect(); + assert!( + collapsed.is_empty(), + "{} absolute anon samples for pid {big} collapsed below {} bytes while it held {peak}: \ + (tid, size) = {:?}", + collapsed.len(), + peak / 4, + &collapsed[..collapsed.len().min(5)] + ); + + // Running max, not the final net: the pages stayed resident, but only since v6.16 + // does trace_sched_process_exit fire before exit_mm, so older kernels charge the + // SIGKILL teardown's rmap removes to a still-bound owner and net the sum to ~0. + let rmap_peak = events + .iter() + .sorted_by_key(|e| e.timestamp) + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 1, delta } if e.pid == big => Some(delta), + _ => None, + }) + .scan(0i64, |net, delta| { + *net += delta; + Some(*net) + }) + .max() + .unwrap_or(0) + * page_size() as i64; + assert!( + rmap_peak >= (peak / 2) as i64, + "fixture never held the region per rmap: peak net {rmap_peak} bytes" + ); + + let forks = events + .iter() + .filter(|e| matches!(e.kind, MemtrackEventKind::Fork { .. })) + .count(); + assert!( + forks >= 200, + "the mm_struct-recycling fork burst did not run tracked: {forks} forks" + ); + + // Every fixture process is dead by now, so no ownership entry may still name one. + let tracked: BTreeSet = events.iter().map(|e| e.pid).collect(); + let residue: Vec = maps + .pid_mm + .iter() + .filter(|(pid, _)| tracked.contains(&(*pid as i32))) + .map(|e| format!("pid_mm {e:?}")) + .chain( + maps.mm_owner + .iter() + .filter(|(_, owner)| tracked.contains(&(*owner as i32))) + .map(|e| format!("mm_owner {e:?}")), + ) + .collect(); + assert!( + residue.is_empty(), + "ownership entries outlived their pid: {residue:?}" + ); + Ok(()) +} + +/// A fork issued by a worker thread must still track the child: registration +/// keys on the parent's tgid (task_newtask fires in the cloning task, whose +/// pid_tgid upper half is the tgid), not on the raw creator tid. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_thread_fork.c"), + "rmap_thread_fork", + shared::track_command_with_rmap, + )?; + + // Single fork in the fixture: parent = the fixture process (tgid), child = + // the region-faulting process. + let (parent, child) = + first_fork_pair(&events).ok_or("no fork event: worker-thread fork was not tracked")?; + assert_ne!(parent, child); + + let child_anon = rmap_bytes(&events, child, None, 1, true); + assert!( + child_anon >= (REGION_MIB - 16) * MIB, + "child of a worker-thread fork missed rmap tracking: anon adds {child_anon} bytes, \ + expected ~{REGION_MIB} MiB" + ); + Ok(()) +} + +/// Both accountings restart from zero at EXEC, so past that point they must agree: +/// rss_stat reports the kernel's own counters while rmap sums folio add/remove +/// deltas, and a lost fork seed or a dropped in-context event shows up as a +/// reconstructed peak far below the counters. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rmap_matches_rss_stat_across_execs() -> Result<(), Box> { + // Two compressors in one shell so ancestor state spans both execs. xz -9's + // dictionary and bzip2 -9's 900k block buffer are what make the peaks large + // enough to compare; the input only has to be compressible, not random. + let temp_dir = TempDir::new()?; + let mut command = Command::new("bash"); + command + .arg("-c") + .arg( + "yes alpha bravo charlie delta | head -c 16777216 > corpus.tar \ + && xz -9 -k -f -T1 corpus.tar && bzip2 -9 -k -f corpus.tar", + ) + .current_dir(temp_dir.path()); + + let (events, thread_handle) = shared::track_command_with_rmap(command)?; + thread_handle.join().unwrap(); + + let (_order, rss, rmap) = per_pid_raw(&events); + + // Only pids that exec'd can be compared: a pid whose pages were already + // resident when the tracker attached has absolute rss_stat counters covering + // them but no rmap events, and nothing resets that deficit. EXEC does. + let exec_pids: BTreeSet = events + .iter() + .filter(|e| matches!(e.kind, MemtrackEventKind::Exec)) + .map(|e| e.pid) + .collect(); + + let mut compared = 0; + for pid in &exec_pids { + // Large enough that page-level noise cannot explain a shortfall. + let rss_peak = rss.get(pid).map_or(0, |acc| acc.max_rss.max(0)) as u64; + if rss_peak <= 6 * MIB { + continue; + } + compared += 1; + let rmap_peak = rmap.get(pid).map_or(0, |acc| acc.max_rss.max(0)) as u64; + assert!( + rmap_peak * 4 >= rss_peak * 3, + "pid {pid}: rmap reconstructed {rmap_peak} bytes against rss_stat's {rss_peak}" + ); + } + assert!( + compared >= 2, + "expected xz and bzip2 above 6 MiB, compared {compared} of {} exec'd pids", + exec_pids.len() + ); + Ok(()) +} + +/// A CLONE_VM child shares its parent's mm until exec. Its faults must not +/// transfer ownership because exec cleanup would remove the parent's live +/// binding. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rmap_clone_vm_keeps_parent_ownership() -> Result<(), Box> { + // Disk-backed: the fixture mmaps a data file next to argv[1] (see build_fixture). + std::fs::create_dir_all(env!("CARGO_TARGET_TMPDIR"))?; + let tmp = TempDir::new_in(env!("CARGO_TARGET_TMPDIR"))?; + write_fixture_headers(tmp.path())?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/vfork_spawn.c"), + "vfork_spawn", + tmp.path(), + )?; + let ready = tmp.path().join("checkpoint-ready"); + let release = tmp.path().join("checkpoint-release"); + let mut command = Command::new(&binary); + command.arg(&ready).arg(&release); + + let (events, checkpoint, root_pid, thread_handle) = + shared::track_command_with_rmap_checkpoint(command, &ready, &release)?; + thread_handle.join().unwrap(); + + // A stale entry naming the parent is insufficient; check its current mm. + let parent_mm = checkpoint + .pid_mm + .iter() + .find_map(|&(pid, mm)| (pid == root_pid as u32).then_some(mm)) + .expect("parent has no pid_mm binding at the checkpoint"); + assert!( + checkpoint.mm_owner.contains(&(parent_mm, root_pid as u32)), + "mm_owner lost the parent's current mm after its CLONE_VM child exec'd \ + (parent mm {parent_mm:#x}, snapshot: {:?})", + checkpoint.mm_owner + ); + + let pairs = fork_pairs(&events); + let [(parent, shared_child), (reclaimer_parent, reclaimer), ..] = pairs[..] else { + return Err("expected forks for the CLONE_VM child and the reclaimer".into()); + }; + assert_eq!( + reclaimer_parent, parent, + "the reclaimer was not forked by the fixture" + ); + + let owner_adds = rmap_bytes(&events, parent, None, 0, true); + assert!( + owner_adds >= 32 * MIB, + "in-context file rmap adds too small: {owner_adds}" + ); + + let shared_mm_adds = rmap_bytes(&events, shared_child, None, 1, true); + assert!( + shared_mm_adds >= 256 * 1024, + "CLONE_VM child faulted no anon pages into the shared mm: {shared_mm_adds} bytes" + ); + assert!( + events + .iter() + .any(|e| e.pid == shared_child && matches!(e.kind, MemtrackEventKind::Exec)), + "CLONE_VM child never exec'd, so its ownership cleanup never ran" + ); + + // process_madvise runs in the reclaimer's context, so removals require + // foreign ownership attribution. + let reclaimed = rmap_bytes(&events, parent, Some(reclaimer), 0, false); + assert!( + reclaimed >= 8 * MIB, + "foreign reclaim was not attributed to the owner after its CLONE_VM child exec'd: only {reclaimed} bytes" + ); + Ok(()) +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index f55cff49b..499d74c53 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -1,12 +1,13 @@ #![allow(dead_code, unused)] +pub use memtrack::OwnershipMaps; use memtrack::prelude::*; use memtrack::{BpfVariant, Tracker}; use runner_shared::artifacts::{MemtrackEvent as Event, MemtrackEventKind}; use std::path::Path; use std::process::Command; -type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; +pub type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; /// Snapshot every tracked event, ordered by timestamp and deduplicated by /// `(addr, kind)` so repeated tracking of one allocation counts once. @@ -18,10 +19,22 @@ type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; macro_rules! assert_events_snapshot { ($name:expr, $events:expr) => {{ use itertools::Itertools; + use runner_shared::artifacts::MemtrackEventKind; use std::mem::discriminant; let formatted_events: Vec = $events .iter() + .filter(|e| { + // RSS and lifecycle events are asserted by dedicated tests, not snapshots. + !matches!( + e.kind, + MemtrackEventKind::Rss { .. } + | MemtrackEventKind::Rmap { .. } + | MemtrackEventKind::Fork { .. } + | MemtrackEventKind::Exec + | MemtrackEventKind::Exit + ) + }) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .map(|e| shared::describe_kind(&e.kind)) @@ -97,6 +110,20 @@ pub fn between_markers(events: &[Event]) -> Vec { events .iter() + // Drop non-allocator events before slicing: the marker window's skip(2) + // is positional (it drops [marker-malloc, marker-free]), so a stray + // event sorting between the pair would displace it and leak the + // marker free. + .filter(|e| { + !matches!( + e.kind, + MemtrackEventKind::Rss { .. } + | MemtrackEventKind::Rmap { .. } + | MemtrackEventKind::Fork { .. } + | MemtrackEventKind::Exec + | MemtrackEventKind::Exit + ) + }) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .skip_while(|e| !is_marker(e)) @@ -147,6 +174,27 @@ pub fn track_binary(binary: &Path) -> TrackResult { track_command(Command::new(binary)) } +pub fn compile_c_source( + source_code: &str, + name: &str, + output_dir: &Path, +) -> Result> { + let source_path = output_dir.join(format!("{name}.c")); + let binary_path = output_dir.join(name); + std::fs::write(&source_path, source_code)?; + + let output = Command::new("gcc") + .args(["-O0", "-o", binary_path.to_str().unwrap()]) + .arg(&source_path) + .output()?; + if !output.status.success() { + error!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); + return Err("Failed to compile C fixture".into()); + } + + Ok(binary_path) +} + /// Track a command, collecting all memory events. No allocators are pre-attached: /// the exec-mapping watcher discovers them as the tracked tree maps executables. pub fn track_command(command: Command) -> TrackResult { @@ -158,6 +206,53 @@ pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> Trac track_command_with_tracker(command, Tracker::with_variant(variant)?) } +/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. +pub fn track_command_with_rmap(command: Command) -> TrackResult { + track_command_with_tracker(command, Tracker::new_without_allocators_with_rmap(true)?) +} + +/// Track a command with rmap hooks and snapshot its ownership maps after the +/// tracked tree exits but before tracker teardown frees the BPF maps. +pub fn track_command_with_rmap_maps( + command: Command, +) -> anyhow::Result<(Vec, OwnershipMaps, std::thread::JoinHandle<()>)> { + let tracker = Tracker::new_without_allocators_with_rmap(true)?; + let (tracker, events, ()) = run_tracked(command, tracker, |_, _| Ok(()))?; + let maps = tracker.ownership_maps()?; + Ok((events, maps, std::thread::spawn(move || drop(tracker)))) +} + +/// Track a command with rmap hooks and snapshot the ownership maps at a +/// fixture-signalled checkpoint. +/// +/// The fixture creates `ready` and waits for `release`. The root pid is returned +/// for correlating snapshot entries with events. +pub fn track_command_with_rmap_checkpoint( + command: Command, + ready: &Path, + release: &Path, +) -> anyhow::Result<(Vec, OwnershipMaps, i32, std::thread::JoinHandle<()>)> { + let tracker = Tracker::new_without_allocators_with_rmap(true)?; + let (tracker, events, (maps, root_pid)) = run_tracked(command, tracker, |tracker, pid| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !ready.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let maps = tracker.ownership_maps()?; + // Release before bailing: nothing else reaps the fixture, which is + // parked holding its mapping. + std::fs::write(release, b"")?; + ensure!(ready.exists(), "fixture never reached the checkpoint"); + Ok((maps, pid)) + })?; + Ok(( + events, + maps, + root_pid, + std::thread::spawn(move || drop(tracker)), + )) +} + /// How many events of each kind-and-size a run saw. Addresses, timestamps, pids /// and event order all differ legitimately between runs of the same workload, so /// none of them can be compared across variants. @@ -166,6 +261,18 @@ type EventProfile = std::collections::BTreeMap; fn event_profile(events: &[Event]) -> EventProfile { let mut profile = EventProfile::new(); for event in events { + // Only allocator events are comparable across variants: RSS and + // lifecycle values (sizes, pids) are per-run. + if !matches!( + event.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) { + continue; + } *profile.entry(describe_kind(&event.kind)).or_default() += 1; } profile @@ -213,10 +320,28 @@ pub fn for_each_variant( } fn track_command_with_tracker(command: Command, tracker: Tracker) -> TrackResult { + let (tracker, events, ()) = run_tracked(command, tracker, |_, _| Ok(()))?; + + // Detaching the probes blocks on RCU grace periods; let the caller decide + // when to wait for it. + let thread_handle = std::thread::spawn(move || drop(tracker)); + + Ok((events, thread_handle)) +} + +/// Run `command` to completion under `tracker` and drain its events, handing +/// back the still-live tracker so BPF state can be read before teardown. +/// `checkpoint` runs while the tracked tree is still live. +fn run_tracked( + command: Command, + tracker: Tracker, + checkpoint: impl FnOnce(&Tracker, i32) -> anyhow::Result, +) -> anyhow::Result<(Tracker, Vec, T)> { tracker.enable_tracking()?; let mut session = tracker.spawn(&command, None)?; let rx = session.take_events()?; + let checkpoint = checkpoint(&tracker, session.pid())?; session.wait()?; // Dropping the session does a final ring buffer drain and closes the @@ -226,12 +351,8 @@ fn track_command_with_tracker(command: Command, tracker: Tracker) -> TrackResult tracker.finish()?; - // Detaching the probes blocks on RCU grace periods; let the caller decide - // when to wait for it. - let thread_handle = std::thread::spawn(move || drop(tracker)); - info!("Tracked {} events", events.len()); trace!("Events: {events:#?}"); - Ok((events, thread_handle)) + Ok((tracker, events, checkpoint)) } diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap new file mode 100644 index 000000000..bd39e6cca --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap new file mode 100644 index 000000000..2745b74d2 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 64, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap new file mode 100644 index 000000000..bc24de5b8 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap @@ -0,0 +1,42 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnonKb:": 64, + "ParentRssAnonKb:": 32 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap new file mode 100644 index 000000000..3a97ec7da --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap @@ -0,0 +1,45 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnon:": 32, + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap new file mode 100644 index 000000000..bd39e6cca --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap new file mode 100644 index 000000000..6e2a2202f --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap new file mode 100644 index 000000000..bd39e6cca --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap new file mode 100644 index 000000000..19bb441e5 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 0, + "RssShmem:": 64 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap new file mode 100644 index 000000000..bd39e6cca --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} From 0b9565ebefe47a4691e3736ab1ac66098d36f1a7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 16:32:49 +0200 Subject: [PATCH 6/9] ci(memtrack): run bpf tests on arm and add rss_tests shard Add an aarch64 lane to the bpf-tests matrix and run the rss integration tests alongside the existing test binaries. --- .github/workflows/ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbbb05a93..f8fe75eac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,14 +85,15 @@ jobs: run: cargo run -- exec -m walltime --skip-upload --warmup-time 0s --max-rounds 5 -- ls -la bpf-tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-22.04, ubuntu-latest, ubuntu-24.04-arm, ubuntu-26.04, ubuntu-26.04-arm] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -100,7 +101,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - cache-key: ${{ matrix.test }} + cache-key: ${{ matrix.os }}-${{ matrix.test }} - uses: ./.github/actions/install-bpf-deps - name: Install additional allocators @@ -109,7 +110,18 @@ jobs: - name: Run tests env: RUST_LOG: debug - run: sudo -E $(which cargo) test --lib --test ${{ matrix.test }} -- --test-threads 1 --nocapture + # Ubuntu 26.04 ships sudo-rs, which ignores `-E`; pass the env the + # rustup shims and the test gate need through `env` instead. + run: | + sudo env \ + "HOME=$HOME" \ + "PATH=$PATH" \ + "CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}" \ + "RUSTUP_HOME=${RUSTUP_HOME:-$HOME/.rustup}" \ + "CARGO_INCREMENTAL=$CARGO_INCREMENTAL" \ + "RUST_LOG=$RUST_LOG" \ + "GITHUB_ACTIONS=$GITHUB_ACTIONS" \ + $(which cargo) test --lib --test ${{ matrix.test }} -- --test-threads 1 --nocapture working-directory: crates/memtrack # Since we ran the tests with sudo, the build artifacts will have root ownership From 916a3ff3891a295eb4461b8d463ff799faa55482 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 18:08:19 +0200 Subject: [PATCH 7/9] fixup! feat(memtrack): track RSS via kmem:rss_stat tracepoint --- crates/memtrack/src/ebpf/c/rss.bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index c0d3925a3..f759582a1 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -93,7 +93,7 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return 0; } - __u32 cur = bpf_get_current_pid_tgid() >> 32; + __u32 cur = current_tgid(); __u64 key = ((__u64)ctx->mm_id << 32) | (__u32)ctx->member; __u64 size = ctx->size; __u32 owner; From df355a0a71e2fee97e72f896405f2d622a9b547d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 18:08:20 +0200 Subject: [PATCH 8/9] fixup! feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks --- crates/memtrack/src/ebpf/c/rss.bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index f759582a1..aef6a1181 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -208,7 +208,7 @@ static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __u64 addr) { __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); struct task_struct* task = bpf_get_current_task_btf(); - __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 pid = current_tgid(); __u32 owner; if ((__u64)BPF_CORE_READ(task, mm) == mm) { From 7704c29fe1c5628dffda34a733bb7289cf2c0257 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Aug 2026 18:08:20 +0200 Subject: [PATCH 9/9] fixup! feat(memtrack): emit fork/exec/exit lifecycle events --- crates/memtrack/src/ebpf/c/rss.bpf.h | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index aef6a1181..65a6cd3d1 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -319,14 +319,13 @@ int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* pag #define CLONE_THREAD 0x00010000 -SEC("tracepoint/task/task_newtask") -int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { - if (ctx->clone_flags & CLONE_THREAD) { +SEC("tp_btf/task_newtask") +int BPF_PROG(tracepoint_task_newtask, struct task_struct* child, __u64 clone_flags) { + if (clone_flags & CLONE_THREAD) { return 0; } - __u64 tid = bpf_get_current_pid_tgid(); - __u32 parent_pid = tid >> 32; + __u32 parent_pid = current_tgid(); if (!is_tracked(parent_pid)) { return 0; } @@ -335,8 +334,14 @@ int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { * tracepoint fires for CLONE_THREAD too and carries only raw task pids, * which would fill the tracking maps with thread tids that no exit path * removes (group death deletes only the tgid). task_newtask fires before - * wake_up_new_task, so registration precedes any event from the child. */ - __u32 child_pid = ctx->pid; + * wake_up_new_task, so registration precedes any event from the child. + * The BTF-typed variant is used because the child's pid must be read in + * the tracker's namespace, which the tracepoint's raw pid field cannot + * give. */ + __u32 child_pid = task_ns_tgid(child); + if (!child_pid) { + return 0; + } track_child(child_pid, parent_pid); SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); @@ -344,7 +349,7 @@ int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { SEC("tracepoint/sched/sched_process_exec") int tracepoint_sched_process_exec(void* ctx) { - __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 pid = current_tgid(); if (!is_tracked(pid)) { return 0; } @@ -362,7 +367,7 @@ int tracepoint_sched_process_exec(void* ctx) { SEC("tracepoint/sched/sched_process_exit") int tracepoint_sched_process_exit(void* ctx) { - __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 pid = current_tgid(); if (!is_tracked(pid)) { return 0; }