fix(tls): count TLS capture losses and fix the gaps they exposed

containers/kernel_counters.go

@@ -1,6 +1,8 @@
 package containers
 
 import (
+	"strconv"
+
 	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -14,20 +16,66 @@ var tlsCiphertextSkippedDesc = prometheus.NewDesc(
 // kernelCounterCollector exports counters the eBPF programs keep in per-CPU
 // maps: socket-level ciphertext events skipped on TLS connections (before the
 // kernel made that distinction, every one reached the L7 parsers as if it were
-// protocol data), and LLM capture chunks lost to a full ring buffer.
+// protocol data), LLM capture chunks and L7 events lost to a full ring buffer,
+// TLS plaintext the hooks could not attribute to a socket, and how the Go TLS
+// hooks found the sockets they did attribute. It also exports what the kernel
+// gave the programs (node_agent_ebpf_info) and their sizes, so a capture gap
+// on one cluster can be told apart from a bug.
 var llmCaptureDropsDesc = prometheus.NewDesc(
 	"node_agent_llm_capture_drops_total",
 	"LLM capture chunks lost in the kernel because the L7 ring buffer was full",
 	nil, nil,
 )
 
+var tlsPlaintextDroppedDesc = prometheus.NewDesc(
+	"node_agent_tls_plaintext_dropped_total",
+	"TLS plaintext seen by a library hook in the kernel but not attributed to a socket, so never captured, by reason. go_fd_unknown includes TLS over in-memory connections (net.Pipe, gRPC bufconn), which have no socket to capture",
+	[]string{"reason"}, nil,
+)
+
+var l7RingbufDropsDesc = prometheus.NewDesc(
+	"node_agent_l7_ringbuf_drops_total",
+	"L7 events lost in the kernel because the L7 ring buffer was full",
+	nil, nil,
+)
+
+var goTLSFdResolvedDesc = prometheus.NewDesc(
+	"node_agent_go_tls_fd_resolved_total",
+	"Go crypto/tls calls whose socket fd was found, by method and net.Conn wrapper depth. itab: the binary's *net.TCPConn itab; socket: a TCP netFD by its fields, confirmed as a TCP socket of that family by the kernel; shape: a TCP netFD by its fields only, as the kernel has no BTF to confirm it with",
+	[]string{"method", "depth"}, nil,
+)
+
+var ebpfInfoDesc = prometheus.NewDesc(
+	"node_agent_ebpf_info",
+	"The eBPF program variant loaded for this kernel, whether the kernel's BTF loaded, and whether the socket struct offsets read from it were set. Without them sockets cannot be read from fds in the kernel, and Go TLS capture through wrapped connections rests on the shape check alone",
+	[]string{"program_variant", "btf", "socket_offsets"}, nil,
+)
+
+var ebpfProgramInstructionsDesc = prometheus.NewDesc(
+	"node_agent_ebpf_program_instructions",
+	"Instructions in each loaded eBPF program after the kernel rewrote it",
+	[]string{"program"}, nil,
+)
+
+var ebpfProgramVerifiedInstructionsDesc = prometheus.NewDesc(
+	"node_agent_ebpf_program_verified_instructions",
+	"Instructions the verifier processed to load each eBPF program, which its complexity limit applies to. Reported by kernels 5.16+",
+	[]string{"program"}, nil,
+)
+
 type kernelCounterCollector struct {
 	tracer *ebpftracer.Tracer
 }
 
 func (c kernelCounterCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- tlsCiphertextSkippedDesc
 	ch <- llmCaptureDropsDesc
+	ch <- tlsPlaintextDroppedDesc
+	ch <- l7RingbufDropsDesc
+	ch <- goTLSFdResolvedDesc
+	ch <- ebpfInfoDesc
+	ch <- ebpfProgramInstructionsDesc
+	ch <- ebpfProgramVerifiedInstructionsDesc
 }
 
 func (c kernelCounterCollector) Collect(ch chan<- prometheus.Metric) {
@@ -40,4 +88,29 @@ func (c kernelCounterCollector) Collect(ch chan<- prometheus.Metric) {
 	if drops, ok := c.tracer.LLMCaptureDrops(); ok {
 		ch <- prometheus.MustNewConstMetric(llmCaptureDropsDesc, prometheus.CounterValue, float64(drops))
 	}
+	if dropped, ok := c.tracer.TLSPlaintextDropped(); ok {
+		for reason, v := range dropped {
+			ch <- prometheus.MustNewConstMetric(tlsPlaintextDroppedDesc, prometheus.CounterValue, float64(v), reason)
+		}
+	}
+	if drops, ok := c.tracer.L7RingbufDrops(); ok {
+		ch <- prometheus.MustNewConstMetric(l7RingbufDropsDesc, prometheus.CounterValue, float64(drops))
+	}
+	if resolved, ok := c.tracer.GoTLSFdResolved(); ok {
+		for _, r := range resolved {
+			ch <- prometheus.MustNewConstMetric(goTLSFdResolvedDesc, prometheus.CounterValue, float64(r.Count), r.Method, strconv.Itoa(r.Depth))
+		}
+	}
+	if info, ok := c.tracer.EBPFInfo(); ok {
+		ch <- prometheus.MustNewConstMetric(ebpfInfoDesc, prometheus.GaugeValue, 1,
+			info.ProgramVariant, strconv.FormatBool(info.KernelBTF), strconv.FormatBool(info.SocketOffsets))
+		for _, p := range info.Programs {
+			if p.Xlated > 0 {
+				ch <- prometheus.MustNewConstMetric(ebpfProgramInstructionsDesc, prometheus.GaugeValue, float64(p.Xlated), p.Program)
+			}
+			if p.Verified > 0 {
+				ch <- prometheus.MustNewConstMetric(ebpfProgramVerifiedInstructionsDesc, prometheus.GaugeValue, float64(p.Verified), p.Program)
+			}
+		}
+	}
 }

containers/l7_self_metrics.go

@@ -1,8 +1,14 @@
 package containers
 
 import (
+	"net"
+	"strconv"
+
+	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
+	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
 )
 
 var (
@@ -182,6 +188,69 @@ var (
 	)
 )
 
+// TLSAttachTotal counts attempts to attach TLS uprobes to a process, by
+// library (go, openssl; "-" when the process could not be registered) and
+// outcome (see ebpftracer.TLSAttachResult). A TLS capture gap used to be
+// visible only in logs at raised verbosity; a non-zero error,
+// attached_no_offsets or not_registered rate is that gap.
+var TLSAttachTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "node_agent_tls_attach_total",
+		Help: "Attempts to attach TLS uprobes to a process, by library and outcome",
+	},
+	[]string{"lib", "result"},
+)
+
+// L7EventsDroppedTotal counts L7 events discarded in the agent before any
+// protocol parsing: the connection or process they belong to was never
+// found, or the retry queue for such events was full. no_ip_socket marks
+// events on sockets the agent does not track (Unix sockets), not a loss. Events lost in the
+// kernel are counted separately (node_agent_l7_ringbuf_drops_total,
+// node_agent_tls_plaintext_dropped_total).
+var L7EventsDroppedTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "node_agent_l7_events_dropped_total",
+		Help: "L7 events dropped in the agent before protocol parsing, by reason, protocol and whether the payload is TLS plaintext",
+	},
+	[]string{"reason", "protocol", "tls"},
+)
+
+type l7DropLogKey struct {
+	container ContainerID
+	reason    string
+	protocol  l7.Protocol
+}
+
+// l7DropLogged keeps one log line per container, reason and protocol: the
+// counter says how many events are dropped, the line says where.
+var l7DropLogged, _ = lru.New[l7DropLogKey, struct{}](4096)
+
+// unknownConnectionReason names why an event's connection could not be found:
+// no_ip_socket when the event carries no IP socket tuple, as for gRPC over a
+// Unix socket, which the agent does not track; unknown_connection otherwise.
+func unknownConnectionReason(si *ebpftracer.SocketInfo) string {
+	if si == nil || !si.Valid {
+		return "no_ip_socket"
+	}
+	return "unknown_connection"
+}
+
+// dropL7Event counts an L7 event dropped before parsing and, the first time
+// for its container, reason and protocol, logs the process and destination.
+// container is empty when the event's process belongs to no known container.
+func dropL7Event(container ContainerID, reason string, pid uint32, fd uint64, req *l7.RequestData, si *ebpftracer.SocketInfo) {
+	L7EventsDroppedTotal.WithLabelValues(reason, protocolLabel(req.Protocol), strconv.FormatBool(req.TLS)).Inc()
+	if ok, _ := l7DropLogged.ContainsOrAdd(l7DropLogKey{container: container, reason: reason, protocol: req.Protocol}, struct{}{}); ok {
+		return
+	}
+	dst := "unknown"
+	if si != nil && si.Valid {
+		dst = net.JoinHostPort(si.DstIP, strconv.Itoa(int(si.DstPort)))
+	}
+	klog.Infof("L7 events dropped before parsing: reason=%s protocol=%s tls=%t container=%s pid=%d fd=%d dst=%s (logged once per container, reason and protocol)",
+		reason, protocolLabel(req.Protocol), req.TLS, container, pid, fd, dst)
+}
+
 // RegisterL7SelfMetrics registers the agent's L7 self-observability counters
 // and wires the l7-package callbacks that increment them.
 func RegisterL7SelfMetrics(reg prometheus.Registerer) {
@@ -196,6 +265,8 @@ func RegisterL7SelfMetrics(reg prometheus.Registerer) {
 		Http2StageTotal,
 		Http2FramesTotal,
 		Http2PayloadSizeTotal,
+		TLSAttachTotal,
+		L7EventsDroppedTotal,
 	)
 	// Hook the HTTP/2 parser's HPACK error path so we get a counter without
 	// l7 having to import prometheus.

containers/metrics.go

@@ -58,6 +58,8 @@ var metrics = struct {
 	PythonThreadLockWaitTime   *prometheus.Desc
 	NodejsEventLoopBlockedTime *prometheus.Desc
 
+	TLSPlaintextDropped *prometheus.Desc
+
 	GpuUsagePercent       *prometheus.Desc
 	GpuMemoryUsagePercent *prometheus.Desc
 
@@ -113,6 +115,8 @@ var metrics = struct {
 	PythonThreadLockWaitTime:   metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
 	NodejsEventLoopBlockedTime: metric("container_nodejs_event_loop_blocked_time_seconds_total", "Total time the Node.js event loop spent blocked"),
 
+	TLSPlaintextDropped: metric("container_tls_plaintext_dropped_total", "TLS plaintext seen by a library hook in the container's processes but not captured because its socket could not be determined, by reason", "reason"),
+
 	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
 	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),
 }