From bd025c66e06fc6f167c3423cab8db3bd37c3e06f Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sun, 6 Sep 2026 11:21:27 +0530 Subject: [PATCH] feat(metrics): measure protocol detection accuracy across all protocols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detection accuracy was only observable for HTTP/2, via the invalid-frame ratio, and only because that ratio happened to be instrumented while chasing a specific bug. That bug turned out to be a weak byte-pattern heuristic whose verdict was cached per connection, so a single false positive poisoned an HTTPS/1.1 connection permanently. Every other protocol is detected the same way. The ClickHouse check is three bytes, and each protocol caches its verdict identically, so the same failure mode is possible for Postgres, Redis, MySQL and Zookeeper and is currently unmeasured. There is no basis for assuming HTTP/2 was unique rather than merely the one investigated. node_agent_protocol_reclassified_total{protocol} node_agent_connections_parsed_total{protocol} A reclassification is a definitive misdetection: eBPF decided the protocol and the parser then refused the payload parseFailThreshold times in a row. Both counters are incremented once per connection — reclassified at the threshold, parsed on first success — so the ratio is a per-connection misdetection rate rather than a per-event one, which is what the HTTP/2 invalid-frame ratio got wrong: one bad connection emitting thousands of invalid frames dominated it. trackParseOK replaces the bare parseFailCount resets at the five success sites, so the denominator is recorded wherever the numerator can be. --- containers/container.go | 23 +++++++++++++++++++---- containers/llm_metrics.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/containers/container.go b/containers/container.go index a8b8d960..10cf3d67 100644 --- a/containers/container.go +++ b/containers/container.go @@ -103,6 +103,7 @@ type ActiveConnection struct { parseFailCount int protocolOverride l7.Protocol // non-zero = override eBPF-detected protocol + parseSucceeded bool // at least one successful parse seen on this connection } type ListenDetails struct { @@ -1292,7 +1293,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 // Empty payloads are normal and are not counted as failures. if len(r.Payload) > 0 { if parser.SawValidFrame() { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } else { c.trackParseFail(conn, pid, fd, r.Protocol) } @@ -1402,7 +1403,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if query == "" && r.Method != l7.MethodStatementClose { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.PostgresQuery(query, r.Status.Error(), r.Duration) @@ -1456,7 +1457,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if query == "" { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.ClickhouseQuery(query, r.Status.Error(), r.Duration) @@ -1468,7 +1469,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if op == "" { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.ZookeeperRequest(op, arg, r.Status, r.Duration) @@ -1497,11 +1498,25 @@ func (c *Container) trackParseFail(conn *ActiveConnection, pid uint32, fd uint64 conn.parseFailCount++ if conn.parseFailCount == parseFailThreshold { conn.protocolOverride = protocolReclassified + // Counted once per connection, at the threshold, so the metric is a + // count of connections given up on rather than of failed parses. + ProtocolReclassifiedTotal.WithLabelValues(protocolLabel(proto)).Inc() klog.Warningf("reclassified connection pid=%d fd=%d from %s to unknown after %d consecutive parse failures", pid, fd, proto, conn.parseFailCount) } } +// trackParseOK records the denominator for the misdetection rate: the first +// successful parse on a connection. Counted once, so both sides of the ratio +// are per-connection rather than per-event. +func (c *Container) trackParseOK(conn *ActiveConnection, proto l7.Protocol) { + if !conn.parseSucceeded { + conn.parseSucceeded = true + ConnectionsParsedTotal.WithLabelValues(protocolLabel(proto)).Inc() + } + conn.parseFailCount = 0 +} + // processHTTP2WithoutConnection handles HTTP/2 events when TCP connection tracking failed. // This is common for Go TLS connections where goroutines switch threads between // TCP connect and TLS write, causing fd_by_pid_tgid lookup to fail in eBPF. diff --git a/containers/llm_metrics.go b/containers/llm_metrics.go index 96f44b8c..4d32773f 100644 --- a/containers/llm_metrics.go +++ b/containers/llm_metrics.go @@ -259,6 +259,38 @@ var ( []string{"bucket", "destination", "direction"}, ) + // ProtocolReclassifiedTotal counts connections given up on after + // parseFailThreshold consecutive parse failures, and ConnectionsParsedTotal + // counts connections that produced at least one successful parse. Together + // they give a per-protocol misdetection rate. + // + // Detection accuracy was previously only observable for HTTP/2, via the + // invalid-frame ratio, and only because that ratio was instrumented while + // chasing a specific bug. Every protocol here is detected by a byte-pattern + // heuristic of similar strength — the ClickHouse check is three bytes — and + // each one caches its verdict on the connection, so the same failure mode + // that made HTTP/2 misclassify HTTPS/1.1 traffic is possible elsewhere and + // currently unmeasured. + // + // A reclassification is a definitive misdetection: the protocol was decided + // by eBPF and then contradicted by the parser refusing the payload + // repeatedly. Rate = reclassified / (reclassified + parsed). + ProtocolReclassifiedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "node_agent_protocol_reclassified_total", + Help: "Connections reclassified after repeated parse failures, by protocol", + }, + []string{"protocol"}, + ) + + ConnectionsParsedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "node_agent_connections_parsed_total", + Help: "Connections that produced at least one successful parse, by protocol", + }, + []string{"protocol"}, + ) + // ContainerLLMCachedTokensTotal counts input tokens served from the // provider's prompt cache. Already counted in token_usage_total{type=input}; // this is a separate metric to make cache-hit rate computable. @@ -330,6 +362,8 @@ func RegisterLLMMetrics(reg prometheus.Registerer) { Http2StageTotal, Http2FramesTotal, Http2PayloadSizeTotal, + ProtocolReclassifiedTotal, + ConnectionsParsedTotal, ContainerLLMCachedTokensTotal, ContainerLLMToolCallsTotal, ContainerLLMCostUSDTotal,