feat(memtrack): capture allocation stacks in eBPF

.github/workflows/ci.yml

@@ -93,7 +93,7 @@ jobs:
         # Each memtrack integration test binary runs its cases serially
         # (eBPF tracker can't overlap with itself in one process), so we
         # shard at the test-binary level to parallelize across jobs.
-        test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests]
+        test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -133,7 +133,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mode: [simulation, walltime]
+        mode: [simulation, walltime, memory]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:

Cargo.toml

@@ -111,6 +111,7 @@ ipc-channel = "0.20"
 itertools = "0.14.0"
 rayon = "1.12"
 linux-perf-event-reader = "0.10.2"                                                       # matches the version linux-perf-data resolves to
+perf-event-open-sys = "6.0"
 env_logger = "0.11.10"
 tempfile = "3.27.0"
 object = { version = "0.39", default-features = false, features = ["read_core", "elf"] }

crates/memtrack/AGENTS.md

@@ -20,11 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab
 
 Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed.
 
+Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.`
+
+### Event stream compatibility
+
+Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream.
+
+This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown.
+
 > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking.
 
 ## Key Directories
 
-- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`.
+- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`.
+- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector.
 - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`.
 - `tests/` — integration tests + `snapshots/` (insta).
 - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces).
@@ -75,7 +84,7 @@ sudo -E cargo test --test c_tests -- --test-threads 1
 - **Build toolchain:** `clang` + BTF/vmlinux headers, `libbpf-dev`, `zlib1g-dev`, `pkgconf`, `build-essential`; vendored libbpf also needs `autopoint`/`bison`/`flex`.
 - `vmlinux.h` is pinned to a specific git rev; `libbpf-rs` uses the `vendored` feature (dist links `libbpf-rs/static`).
 
-Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate).
+Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_MEMTRACK_TRACK_ALLOCATORS` (0/false disables), `CODSPEED_MEMTRACK_TRACK_RMAP` (1 enables), `CODSPEED_MEMTRACK_CAPTURE_STACKS` (0/false disables), `CODSPEED_MEMTRACK_STACK_BUDGET` (stack copy size in bytes, default 8192), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate).
 
 ## Testing & QA
 

crates/memtrack/Cargo.toml

@@ -16,7 +16,7 @@ required-features = ["ebpf"]
 
 [features]
 default = ["ebpf"]
-ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"]
+ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux", "dep:mimalloc"]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -34,9 +34,11 @@ itertools = { workspace = true }
 paste = "1.0.15"
 libbpf-rs = { version = "0.26", features = ["vendored"], optional = true }
 object = { workspace = true }
+perf-event-open-sys = { workspace = true }
 rayon = "1.12"
 parking_lot = "0.12"
 typed-builder = "0.23.2"
+mimalloc = { version = "0.1", optional = true }
 
 [build-dependencies]
 libbpf-cargo = { version = "0.26", optional = true }

crates/memtrack/src/ebpf/c/allocator.h

@@ -9,6 +9,7 @@
     BPF_HASH_MAP(name##_arg, __u64, __u64, 10000);   \
     SEC(UPROBE_SEC)                                  \
     int uprobe_##name(struct pt_regs* ctx) {         \
+        stash_stack_hash(capture_stack(ctx));        \
         return store_param(&name##_arg, arg_expr);   \
     }                                                \
     SEC(URETPROBE_SEC)                               \
@@ -17,6 +18,7 @@
         if (!arg_ptr) {                              \
             return 0;                                \
         }                                            \
+        __u64 stack_hash = take_stack_hash();        \
         __u64 ret_val = PT_REGS_RC(ctx);             \
         if (ret_val == 0) {                          \
             return 0;                                \
@@ -32,6 +34,7 @@
         if (arg0 == 0) {                         \
             return 0;                            \
         }                                        \
+        __u64 stack_hash = capture_stack(ctx);   \
         submit_block;                            \
     }
 
@@ -50,6 +53,8 @@
             return 0;                                                         \
         }                                                                     \
                                                                               \
+        stash_stack_hash(capture_stack(ctx));                                 \
+                                                                              \
         struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr};   \
                                                                               \
         bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY);              \
@@ -63,6 +68,7 @@
         if (!args) {                                                          \
             return 0;                                                         \
         }                                                                     \
+        __u64 stack_hash = take_stack_hash();                                 \
                                                                               \
         struct name##_args_t a = *args;                                       \
         bpf_map_delete_elem(&name##_args, &tid);                              \
@@ -77,20 +83,22 @@
         submit_block;                                                         \
     }
 
-UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); })
+UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx),
+               { return submit_alloc_event(arg0, ret_val, stack_hash); })
 
-UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); })
+UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); })
 
 UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx),
-               { return submit_calloc_event(arg0, ret_val); })
+               { return submit_calloc_event(arg0, ret_val, stack_hash); })
 
 UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx),
-                { return submit_realloc_event(arg1, ret_val, arg0); })
+                { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); })
 
 UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx),
-               { return submit_aligned_alloc_event(arg0, ret_val); })
+               { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); })
 
-UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); })
+UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx),
+               { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); })
 
 /*
  * posix_memalign(void** memptr, size_t alignment, size_t size)
@@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) {
         return 0;
     }
 
+    stash_stack_hash(capture_stack(ctx));
+
     struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)};
     bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY);
     return 0;
@@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) {
     if (!args) {
         return 0;
     }
+    __u64 stack_hash = take_stack_hash();
 
     struct posix_memalign_args_t a = *args;
     bpf_map_delete_elem(&posix_memalign_args, &tid);
@@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) {
         return 0;
     }
 
-    return submit_aligned_alloc_event(a.size, addr);
+    return submit_aligned_alloc_event(a.size, addr, stack_hash);
 }
 
 struct mmap_args {

crates/memtrack/src/ebpf/c/event.h

@@ -15,6 +15,49 @@
 #define EVENT_TYPE_RSS 12
 #define EVENT_TYPE_RMAP 13
 
+/* Largest user-stack copy one definition can carry. Every capture reserves
+ * header + budget in the ring up front, so this bounds ring space held per
+ * in-flight capture, per-capture copy cost, and the verifier work per load
+ * (which scales with the frozen budget). The kernel itself allows records up
+ * to the ring size; this is a policy cap. */
+#define MEMTRACK_MAX_STACK_COPY (32 * 1024)
+
+/* Registers, indexed by the capturing architecture's DWARF register number
+ * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30,
+ * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline
+ * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just
+ * ip/sp/bp. */
+#define MEMTRACK_STACK_REGS 33
+
+/* Counter slots in the stack_counters array map. */
+#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0
+#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1
+/* bpf_get_stackid() has several negative outcomes (no user callchain,
+ * hash-bucket collision, or no free bucket), so this counts only missing ids. */
+#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2
+#define MEMTRACK_STACK_COUNTER_TRUNCATED 3
+#define MEMTRACK_STACK_COUNTER_RING_FULL 4
+#define MEMTRACK_STACK_COUNTER_COUNT 5
+
+struct stack_regs {
+    uint64_t reg[MEMTRACK_STACK_REGS];
+};
+
+/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp`
+ * follow it. */
+struct stack_header {
+    uint64_t hash;
+    uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */
+    int64_t stackid;    /* bpf_get_stackid() result; negative means unavailable */
+    uint64_t sp;        /* user stack pointer the copy starts at */
+    uint32_t pid;
+    uint32_t tid;
+    uint32_t copy_len;
+    uint8_t truncated; /* the copy hit the size cap */
+    uint8_t _pad[3];
+    struct stack_regs regs;
+};
+
 /* Common header shared by all event types */
 struct event_header {
     uint8_t event_type; /* See EVENT_TYPE_* constants above */
@@ -29,20 +72,23 @@ struct event {
     union {
         /* Allocation events (malloc, calloc, aligned_alloc) */
         struct {
-            uint64_t addr; /* address returned */
-            uint64_t size; /* size requested */
+            uint64_t addr;       /* address returned */
+            uint64_t size;       /* size requested */
+            uint64_t stack_hash; /* caller stack identity; 0 = not captured */
         } alloc;
 
         /* Deallocation event (free) */
         struct {
-            uint64_t addr; /* address to free */
+            uint64_t addr;       /* address to free */
+            uint64_t stack_hash; /* caller stack identity; 0 = not captured */
         } free;
 
         /* Reallocation event - includes both old and new addresses */
         struct {
-            uint64_t old_addr; /* previous address (can be NULL) */
-            uint64_t new_addr; /* new address returned */
-            uint64_t size;     /* new size requested */
+            uint64_t old_addr;   /* previous address (can be NULL) */
+            uint64_t new_addr;   /* new address returned */
+            uint64_t size;       /* new size requested */
+            uint64_t stack_hash; /* caller stack identity; 0 = not captured */
         } realloc;
 
         /* Memory mapping events (mmap, munmap, brk) */

crates/memtrack/src/ebpf/c/main.bpf.c

@@ -11,6 +11,7 @@
 #include "process_tracking.bpf.h"
 #include "rmap.bpf.h"
 #include "rss.bpf.h"
+#include "stack_capture.bpf.h"
 #include "utils/event_helpers.h"
 #include "utils/folio.h"
 #include "utils/map_helpers.h"