diff --git a/containers/container.go b/containers/container.go index a8b8d960..10cf3d67 100644 --- a/containers/container.go +++ b/containers/container.go @@ -103,6 +103,7 @@ type ActiveConnection struct { parseFailCount int protocolOverride l7.Protocol // non-zero = override eBPF-detected protocol + parseSucceeded bool // at least one successful parse seen on this connection } type ListenDetails struct { @@ -1292,7 +1293,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 // Empty payloads are normal and are not counted as failures. if len(r.Payload) > 0 { if parser.SawValidFrame() { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } else { c.trackParseFail(conn, pid, fd, r.Protocol) } @@ -1402,7 +1403,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if query == "" && r.Method != l7.MethodStatementClose { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.PostgresQuery(query, r.Status.Error(), r.Duration) @@ -1456,7 +1457,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if query == "" { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.ClickhouseQuery(query, r.Status.Error(), r.Duration) @@ -1468,7 +1469,7 @@ func (c *Container) onL7RequestWithResult(pid uint32, fd uint64, timestamp uint6 if op == "" { c.trackParseFail(conn, pid, fd, r.Protocol) } else { - conn.parseFailCount = 0 + c.trackParseOK(conn, r.Protocol) } if trace != nil { trace.ZookeeperRequest(op, arg, r.Status, r.Duration) @@ -1497,11 +1498,25 @@ func (c *Container) trackParseFail(conn *ActiveConnection, pid uint32, fd uint64 conn.parseFailCount++ if conn.parseFailCount == parseFailThreshold { conn.protocolOverride = protocolReclassified + // Counted once per connection, at the threshold, so the metric is a + // count of connections given up on rather than of failed parses. + ProtocolReclassifiedTotal.WithLabelValues(protocolLabel(proto)).Inc() klog.Warningf("reclassified connection pid=%d fd=%d from %s to unknown after %d consecutive parse failures", pid, fd, proto, conn.parseFailCount) } } +// trackParseOK records the denominator for the misdetection rate: the first +// successful parse on a connection. Counted once, so both sides of the ratio +// are per-connection rather than per-event. +func (c *Container) trackParseOK(conn *ActiveConnection, proto l7.Protocol) { + if !conn.parseSucceeded { + conn.parseSucceeded = true + ConnectionsParsedTotal.WithLabelValues(protocolLabel(proto)).Inc() + } + conn.parseFailCount = 0 +} + // processHTTP2WithoutConnection handles HTTP/2 events when TCP connection tracking failed. // This is common for Go TLS connections where goroutines switch threads between // TCP connect and TLS write, causing fd_by_pid_tgid lookup to fail in eBPF. diff --git a/containers/llm_metrics.go b/containers/llm_metrics.go index 96f44b8c..4d32773f 100644 --- a/containers/llm_metrics.go +++ b/containers/llm_metrics.go @@ -259,6 +259,38 @@ var ( []string{"bucket", "destination", "direction"}, ) + // ProtocolReclassifiedTotal counts connections given up on after + // parseFailThreshold consecutive parse failures, and ConnectionsParsedTotal + // counts connections that produced at least one successful parse. Together + // they give a per-protocol misdetection rate. + // + // Detection accuracy was previously only observable for HTTP/2, via the + // invalid-frame ratio, and only because that ratio was instrumented while + // chasing a specific bug. Every protocol here is detected by a byte-pattern + // heuristic of similar strength — the ClickHouse check is three bytes — and + // each one caches its verdict on the connection, so the same failure mode + // that made HTTP/2 misclassify HTTPS/1.1 traffic is possible elsewhere and + // currently unmeasured. + // + // A reclassification is a definitive misdetection: the protocol was decided + // by eBPF and then contradicted by the parser refusing the payload + // repeatedly. Rate = reclassified / (reclassified + parsed). + ProtocolReclassifiedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "node_agent_protocol_reclassified_total", + Help: "Connections reclassified after repeated parse failures, by protocol", + }, + []string{"protocol"}, + ) + + ConnectionsParsedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "node_agent_connections_parsed_total", + Help: "Connections that produced at least one successful parse, by protocol", + }, + []string{"protocol"}, + ) + // ContainerLLMCachedTokensTotal counts input tokens served from the // provider's prompt cache. Already counted in token_usage_total{type=input}; // this is a separate metric to make cache-hit rate computable. @@ -330,6 +362,8 @@ func RegisterLLMMetrics(reg prometheus.Registerer) { Http2StageTotal, Http2FramesTotal, Http2PayloadSizeTotal, + ProtocolReclassifiedTotal, + ConnectionsParsedTotal, ContainerLLMCachedTokensTotal, ContainerLLMToolCallsTotal, ContainerLLMCostUSDTotal,