feat(tls): attribute TLS losses per container and find sockets behind wrapped Go connections

containers/container.go

@@ -156,6 +156,9 @@ type Container struct {
 
 	nodejsStats *ebpftracer.NodejsStats
 	pythonStats *ebpftracer.PythonStats
+	// tlsDrops counts TLS plaintext the kernel could not attribute to a
+	// socket for this container's processes, by reason. Guarded by lock.
+	tlsDrops map[string]float64
 
 	mounts     map[string]proc.MountInfo
 	seenMounts map[uint64]struct{}
@@ -499,6 +502,10 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	// registry's event loop; read under it here.
 	c.lock.RLock()
 	pythonStats, nodejsStats := c.pythonStats, c.nodejsStats
+	tlsDrops := make(map[string]float64, len(c.tlsDrops))
+	for reason, n := range c.tlsDrops {
+		tlsDrops[reason] = n
+	}
 	var pythonLockWait, nodejsBlocked float64
 	if pythonStats != nil {
 		pythonLockWait = pythonStats.ThreadLockWaitTime.Seconds()
@@ -513,6 +520,9 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	if nodejsStats != nil {
 		ch <- c.counter(metrics.NodejsEventLoopBlockedTime, nodejsBlocked)
 	}
+	for reason, n := range tlsDrops {
+		ch <- c.counter(metrics.TLSPlaintextDropped, n, reason)
+	}
 
 	// --- L7 metrics: push-model, own lock ---
 	c.l7Stats.collect(ch)
@@ -591,12 +601,20 @@ func (c *Container) onProcessExit(pid uint32, oomKill bool) {
 	}
 }
 
-// closeProcess releases everything held for a process that is gone.
+// closeProcess releases everything held for a process that is gone. The
+// caller holds c.lock.
 func (c *Container) closeProcess(pid uint32, p *Process) {
 	p.Close()
 	if p.golang() {
 		c.registry.tracer.ReleaseGoTLSOffsets(pid)
 	}
+	if p.tlsAttached {
+		// The last chance to attribute its losses: the periodic read comes
+		// too late for a short-lived process.
+		if d := c.registry.tracer.TLSPlaintextDroppedForPid(pid); len(d) > 0 {
+			c.recordTLSDropsLocked(pid, p, d)
+		}
+	}
 }
 
 func (c *Container) onFileOpen(pid uint32, fd uint64, mnt uint64, log bool) {
@@ -2159,6 +2177,7 @@ func (c *Container) attachTlsUprobes(tracer *ebpftracer.Tracer, pid uint32, newS
 	}
 	if !p.goTlsUprobesChecked {
 		p.tlsExe, _ = exeIdentityOf(pid)
+		p.tlsExeName, _ = os.Readlink(proc.Path(pid, "exe"))
 		p.tlsExeCheckedAt = time.Now()
 		links, isGolangApp, result := tracer.AttachGoTlsUprobes(pid)
 		countTLSAttach("go", result)

containers/metrics.go

@@ -58,6 +58,8 @@ var metrics = struct {
 	PythonThreadLockWaitTime   *prometheus.Desc
 	NodejsEventLoopBlockedTime *prometheus.Desc
 
+	TLSPlaintextDropped *prometheus.Desc
+
 	GpuUsagePercent       *prometheus.Desc
 	GpuMemoryUsagePercent *prometheus.Desc
 
@@ -113,6 +115,8 @@ var metrics = struct {
 	PythonThreadLockWaitTime:   metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
 	NodejsEventLoopBlockedTime: metric("container_nodejs_event_loop_blocked_time_seconds_total", "Total time the Node.js event loop spent blocked"),
 
+	TLSPlaintextDropped: metric("container_tls_plaintext_dropped_total", "TLS plaintext seen by a library hook in the container's processes but not captured because its socket could not be determined, by reason", "reason"),
+
 	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
 	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),
 }

containers/process.go

@@ -58,6 +58,7 @@ type Process struct {
 	openSslLastCheck      time.Time
 	tlsAttached           bool
 	tlsExe                exeIdentity
+	tlsExeName            string
 	tlsExeCheckedAt       time.Time
 
 	// Only touched by the instrument goroutine.

containers/registry.go

@@ -718,6 +718,17 @@ func (r *Registry) updateEbpfStatsAndActiveConns() {
 	if !r.tracer.Ready() {
 		return
 	}
+	// TLS plaintext the kernel could not attribute to a socket, per process:
+	// counted on the process's container and logged once per binary.
+	for pid, byReason := range r.tracer.TLSPlaintextDroppedByPid() {
+		r.containerLock.RLock()
+		c := r.containersByPid[pid]
+		r.containerLock.RUnlock()
+		if c != nil {
+			c.recordTLSDrops(pid, byReason)
+		}
+	}
+
 	// Traffic stats from eBPF maps
 	iter := r.tracer.ActiveConnectionsIterator()
 	cid := ebpftracer.ConnectionId{}

containers/tls_drops.go

@@ -0,0 +1,57 @@
+package containers
+
+import (
+	lru "github.com/hashicorp/golang-lru/v2"
+	"k8s.io/klog/v2"
+)
+
+// tlsDropHints explains each tls_plaintext_dropped reason in the log line.
+var tlsDropHints = map[string]string{
+	"go_fd_unknown":       "crypto/tls runs over a connection whose socket the probe cannot find: a net.Conn wrapped by the application or a library, or an in-memory one (net.Pipe, gRPC bufconn)",
+	"ssl_read_fd_unknown": "SSL_read returned data before the connection's socket was seen, as in a server that reads before it writes through a memory BIO",
+	"ssl_write_unclaimed": "SSL_write plaintext was never matched to a socket write, as in a memory-BIO application that buffers several writes before sending",
+}
+
+type tlsDropLogKey struct {
+	exe    exeIdentity
+	reason string
+	// container is set only when the process is unknown, so that an unknown
+	// process in one container does not silence those in every other.
+	container ContainerID
+}
+
+// tlsDropLogged keeps the TLS-drop warning to one line per binary and reason,
+// however many processes run that binary.
+var tlsDropLogged, _ = lru.New[tlsDropLogKey, struct{}](4096)
+
+// recordTLSDropsLocked adds TLS plaintext the kernel could not attribute to a
+// socket to the container's count, and names the binary in the log the first
+// time it happens for it. The caller holds c.lock.
+func (c *Container) recordTLSDropsLocked(pid uint32, p *Process, byReason map[string]uint64) {
+	if c.tlsDrops == nil {
+		c.tlsDrops = map[string]float64{}
+	}
+	for reason, n := range byReason {
+		c.tlsDrops[reason] += float64(n)
+		key := tlsDropLogKey{reason: reason}
+		name := ""
+		if p != nil {
+			key.exe, name = p.tlsExe, p.tlsExeName
+		} else {
+			key.container = c.id
+		}
+		if ok, _ := tlsDropLogged.ContainsOrAdd(key, struct{}{}); ok {
+			continue
+		}
+		klog.Warningf("TLS plaintext of %s (pid %d, container %s) is not being captured: %d %s events. %s",
+			name, pid, c.id, n, reason, tlsDropHints[reason])
+	}
+}
+
+// recordTLSDrops is recordTLSDropsLocked for a process that may still be
+// running, as found by the periodic read.
+func (c *Container) recordTLSDrops(pid uint32, byReason map[string]uint64) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.recordTLSDropsLocked(pid, c.processes[pid], byReason)
+}

ebpftracer/ebpf/l7/gotls.c

@@ -102,6 +102,30 @@ int extract_fd_from_tcpconn(void* conn_data, struct go_tls_offsets *offsets, __u
     return 0;
 }
 
+// GO_CONN_MAX_DEPTH bounds how many net.Conn wrappers are unwrapped.
+#define GO_CONN_MAX_DEPTH 4
+
+// go_plausible_iface reports whether i looks like a non-nil Go interface
+// value: both words user-space pointers rather than small integers.
+static inline __attribute__((__always_inline__))
+int go_plausible_iface(struct go_interface *i) {
+    return (__u64)i->type > 4096 && (__u64)i->ptr > 4096;
+}
+
+// go_fd_is_socket reports whether fd is an IPv4 or IPv6 socket of the
+// current process. It also reports 1 when that cannot be checked (no socket
+// struct offsets), which keeps the previous behaviour of trusting the read.
+static inline __attribute__((__always_inline__))
+int go_fd_is_socket(__u32 fd) {
+    __u32 zero = 0;
+    struct socket_info_offsets *so = bpf_map_lookup_elem(&socket_info_offsets_map, &zero);
+    if (!so || !so->offsets_valid) {
+        return 1;
+    }
+    struct socket_tuple t = {};
+    return get_socket_tuple_from_fd(fd, &t);
+}
+
 static inline __attribute__((__always_inline__))
 int go_crypto_tls_get_fd_from_conn(struct pt_regs *ctx, __u32 *fd) {
     __u64 pid_tgid = bpf_get_current_pid_tgid();
@@ -135,64 +159,36 @@ int go_crypto_tls_get_fd_from_conn(struct pt_regs *ctx, __u32 *fd) {
     }
     bpf_printk("go_tls: conn_interface.type=%llx data=%p", conn_interface.type, conn_interface.ptr);
 
-    // Step 3: Check if this is a gRPC syscallConn wrapper and unwrap if needed
-    // gRPC wraps connections in credentials.syscallConn which implements net.Conn
-    // Structure: syscallConn { Conn net.Conn; rawConn syscall.RawConn }
-    void* actual_conn_data = conn_interface.ptr;
-
-    if (offsets && offsets->grpc_syscallconn_itab != 0) {
-        // Check if this connection is wrapped in gRPC's syscallConn
-        if ((__u64)conn_interface.type == offsets->grpc_syscallconn_itab) {
-            bpf_printk("go_tls: detected gRPC syscallConn wrapper, unwrapping");
-
-            // Read the underlying Conn interface from syscallConn
-            // syscallConn.Conn is at offset syscallconn_conn_offset (typically 0)
-            struct go_interface inner_conn;
-            __s32 sc_offset = offsets->syscallconn_conn_offset;
-            if (sc_offset == 0) {
-                sc_offset = DEFAULT_SYSCALLCONN_CONN_OFFSET;
-            }
-
-            if (bpf_probe_read(&inner_conn, sizeof(inner_conn), conn_interface.ptr + sc_offset)) {
-                bpf_printk("go_tls: failed to read inner conn from syscallConn+%d", sc_offset);
-                return 1;
-            }
-            bpf_printk("go_tls: unwrapped inner_conn.type=%llx data=%p", inner_conn.type, inner_conn.ptr);
-
-            // Use the unwrapped connection data
-            actual_conn_data = inner_conn.ptr;
+    // Step 3: Find the socket behind the connection. Applications and
+    // libraries wrap net.Conn in their own types, each embedding the
+    // connection it wraps as an interface field: first (offset 0), or after a
+    // small counter or flag (offset 8). Proxies stack them; traefik's TLS
+    // connections are TLSConn -> peekConn -> trackedConnection ->
+    // *net.TCPConn, and VictoriaMetrics' scrape connections are statConn
+    // {closed int32; net.Conn}. gRPC's syscallConn is the same shape.
+    // Descend until a level is a TCP connection.
+    __u64 tcp_itab = offsets ? offsets->net_tcpconn_itab : 0;
+    struct go_interface c = conn_interface;
+#pragma unroll
+    for (int depth = 0; depth < GO_CONN_MAX_DEPTH; depth++) {
+        if (tcp_itab && (__u64)c.type == tcp_itab) {
+            // Exact: the binary's own *net.TCPConn itab.
+            return extract_fd_from_tcpconn(c.ptr, offsets, fd);
         }
-    } else {
-        // No itab info available - try heuristic detection
-        // If FD extraction fails with direct approach, try treating as syscallConn
-        // This is a fallback for when we don't have symbol information
-        bpf_printk("go_tls: no itab info, trying direct extraction first");
-    }
-
-    // Step 4: Extract FD from the actual connection data
-    if (extract_fd_from_tcpconn(actual_conn_data, offsets, fd) == 0) {
-        bpf_printk("go_tls: extracted fd=%d", *fd);
-        return 0;
-    }
-
-    // Step 5: If direct extraction failed and we haven't tried unwrapping,
-    // try treating conn_interface.ptr as a wrapper struct
-    if (actual_conn_data == conn_interface.ptr) {
-        bpf_printk("go_tls: direct extraction failed, trying wrapper unwrap");
-
-        // Try reading as if it's a wrapper with Conn interface at offset 0
-        struct go_interface wrapper_inner;
-        if (bpf_probe_read(&wrapper_inner, sizeof(wrapper_inner), conn_interface.ptr) == 0) {
-            if (wrapper_inner.ptr != NULL && wrapper_inner.type != 0) {
-                bpf_printk("go_tls: found wrapper inner.type=%llx data=%p",
-                          wrapper_inner.type, wrapper_inner.ptr);
-
-                if (extract_fd_from_tcpconn(wrapper_inner.ptr, offsets, fd) == 0) {
-                    bpf_printk("go_tls: extracted fd=%d via wrapper unwrap", *fd);
-                    return 0;
-                }
+        // Without the itab (a stripped binary) or with one that does not
+        // match (a PIE binary, whose symbol table holds it unrelocated), try
+        // the level as a TCP connection, and believe the result only if it
+        // names a socket of this process.
+        if (extract_fd_from_tcpconn(c.ptr, offsets, fd) == 0 && go_fd_is_socket(*fd)) {
+            return 0;
+        }
+        struct go_interface inner = {};
+        if (bpf_probe_read(&inner, sizeof(inner), c.ptr) || !go_plausible_iface(&inner)) {
+            if (bpf_probe_read(&inner, sizeof(inner), c.ptr + 8) || !go_plausible_iface(&inner)) {
+                break;
             }
         }
+        c = inner;
     }
 
     bpf_printk("go_tls: failed to extract fd");
@@ -252,7 +248,7 @@ int go_crypto_tls_write_enter(struct pt_regs *ctx) {
 
     __u32 fd;
     if (go_crypto_tls_get_fd_from_conn(ctx, &fd)) {
-        count_tls_drop(TLS_DROP_GO_FD_UNKNOWN);
+        count_tls_drop_by_pid(TLS_DROP_GO_FD_UNKNOWN);
         return 0;
     }
 
@@ -274,7 +270,7 @@ int go_crypto_tls_read_enter(struct pt_regs *ctx) {
 
     __u32 fd;
     if (go_crypto_tls_get_fd_from_conn(ctx, &fd)) {
-        count_tls_drop(TLS_DROP_GO_FD_UNKNOWN);
+        count_tls_drop_by_pid(TLS_DROP_GO_FD_UNKNOWN);
         return 0;
     }
     char *buf_ptr = (char*)GO_PARAM2(ctx);

ebpftracer/ebpf/l7/l7.c

@@ -377,6 +377,39 @@ void count_tls_drop(__u32 reason) {
     }
 }
 
+// tls_plaintext_dropped_by_pid attributes the same losses to a process, so
+// userspace can name the container and binary they come from. The per-CPU
+// total above cannot. Userspace reads and deletes the entries periodically.
+struct tls_drop_key {
+    __u32 pid;
+    __u32 reason;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_LRU_HASH);
+    __uint(key_size, sizeof(struct tls_drop_key));
+    __uint(value_size, sizeof(__u64));
+    __uint(max_entries, 4096);
+} tls_plaintext_dropped_by_pid SEC(".maps");
+
+// count_tls_drop_by_pid also attributes a loss to the current process. It is
+// used only from the TLS library uprobes, which are small, and kept out of
+// the syscall programs, which are already near the verifier's limits.
+static inline __attribute__((__always_inline__))
+void count_tls_drop_by_pid(__u32 reason) {
+    count_tls_drop(reason);
+    struct tls_drop_key k = {};
+    k.pid = bpf_get_current_pid_tgid() >> 32;
+    k.reason = reason;
+    __u64 *v = bpf_map_lookup_elem(&tls_plaintext_dropped_by_pid, &k);
+    if (v) {
+        __sync_fetch_and_add(v, 1);
+        return;
+    }
+    __u64 one = 1;
+    bpf_map_update_elem(&tls_plaintext_dropped_by_pid, &k, &one, BPF_NOEXIST);
+}
+
 // mark_tls records that a TLS library hook handles this connection.
 //
 // A TLS connection is seen twice: once as plaintext by the Go crypto/tls or

ebpftracer/ebpf/l7/openssl.c

@@ -33,7 +33,7 @@ int openssl_SSL_write_enter(struct pt_regs *ctx) {
     // thread names the fd (sys_enter_write and friends in l7.c). A pending
     // write still here was never claimed, and its plaintext is lost.
     if (bpf_map_lookup_elem(&ssl_write_pending, &tid)) {
-        count_tls_drop(TLS_DROP_SSL_WRITE_UNCLAIMED);
+        count_tls_drop_by_pid(TLS_DROP_SSL_WRITE_UNCLAIMED);
     }
     struct ssl_args args = {};
     args.buf = buf;
@@ -88,7 +88,7 @@ int openssl_SSL_read_exit(struct pt_regs *ctx) {
         fd = ssl_known_fd(pid, ssl);
         if (!fd) {
             if ((int)PT_REGS_RC(ctx) > 0) {
-                count_tls_drop(TLS_DROP_SSL_READ_FD_UNKNOWN);
+                count_tls_drop_by_pid(TLS_DROP_SSL_READ_FD_UNKNOWN);
             }
             return 0;
         }