From aed40d374128f392b2e3b08fc9d23813da051dc1 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Sat, 3 Oct 2026 12:24:20 +0000 Subject: [PATCH 01/23] Egress Policy Cache Signed-off-by: Yan Avlasov --- .github/workflows/pr-workflow.yaml | 4 +- cmd/dataplane/envoy/Dockerfile | 17 +- .../envoy/dynamic-modules/.gitignore | 3 + .../{egress-policy => }/Cargo.lock | 9 + .../envoy/dynamic-modules/Cargo.toml | 20 ++ .../egress-policy-cache/Cargo.toml | 31 ++++ .../egress-policy-cache/src/lib.rs | 175 ++++++++++++++++++ .../dynamic-modules/egress-policy/.gitignore | 1 - internal/e2e/collector_metrics.go | 110 +++++++++++ internal/e2e/collector_metrics_test.go | 55 ++++++ .../e2e/suites/egressmitm/egressmitm_test.go | 31 +++- internal/portforward/portforward.go | 7 +- 12 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 cmd/dataplane/envoy/dynamic-modules/.gitignore rename cmd/dataplane/envoy/dynamic-modules/{egress-policy => }/Cargo.lock (98%) create mode 100644 cmd/dataplane/envoy/dynamic-modules/Cargo.toml create mode 100644 cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml create mode 100644 cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs delete mode 100644 cmd/dataplane/envoy/dynamic-modules/egress-policy/.gitignore diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index cdb0436a10..ad6008379f 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -87,8 +87,8 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - cmd/dataplane/envoy/dynamic-modules/*/target - key: cargo-${{ runner.os }}-${{ hashFiles('cmd/dataplane/envoy/dynamic-modules/*/Cargo.lock') }} + cmd/dataplane/envoy/dynamic-modules/target + key: cargo-${{ runner.os }}-${{ hashFiles('cmd/dataplane/envoy/dynamic-modules/Cargo.lock') }} restore-keys: cargo-${{ runner.os }}- - name: dynamic module tests run: | diff --git a/cmd/dataplane/envoy/Dockerfile b/cmd/dataplane/envoy/Dockerfile index 9de411563c..23be3766fb 100644 --- a/cmd/dataplane/envoy/Dockerfile +++ b/cmd/dataplane/envoy/Dockerfile @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Stage 1: Build the release version of the Rust dynamic module +# Stage 1: Build the release version of the Rust dynamic modules FROM rust:bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e AS builder -WORKDIR /usr/src/dynamic-modules/egress-policy +WORKDIR /usr/src/dynamic-modules # Install build dependencies required by bindgen for generating Envoy ABI bindings RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -23,21 +23,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libclang-dev \ && rm -rf /var/lib/apt/lists/* -# Copy rust module source code -COPY dynamic-modules/egress-policy/ . +# Copy rust module workspace source code +COPY dynamic-modules/ . -# Build release version of the dynamic module shared library +# Build release version of the dynamic module shared libraries RUN cargo build --release --locked -# Stage 2: Create the runtime image containing Envoy and the built dynamic module shared library +# Stage 2: Create the runtime image containing Envoy and the built dynamic module shared libraries # When changing Envoy version, the version of SDK in the Cargo.toml files SHOULD match. # Pin the multi-platform image index, not one platform's manifest, so that the # --platform the installer passes (the host architecture on kind, linux/amd64 # otherwise) selects a matching Envoy binary. FROM envoyproxy/envoy:v1.39.1@sha256:57e14a549d7bd43c8d3f6d03e8cfa653e037d4b38e133acd9b54f38c524401b4 -# Copy the built dynamic module shared library from the builder stage -COPY --from=builder /usr/src/dynamic-modules/egress-policy/target/release/libenvoy_substrate_egress_policy.so /usr/local/lib/libenvoy_substrate_egress_policy.so +# Copy the built dynamic module shared libraries from the builder stage +COPY --from=builder /usr/src/dynamic-modules/target/release/libenvoy_substrate_egress_policy.so /usr/local/lib/libenvoy_substrate_egress_policy.so +COPY --from=builder /usr/src/dynamic-modules/target/release/libenvoy_substrate_egress_policy_cache.so /usr/local/lib/libenvoy_substrate_egress_policy_cache.so # Set default search path for Envoy dynamic modules ENV ENVOY_DYNAMIC_MODULES_SEARCH_PATH=/usr/local/lib diff --git a/cmd/dataplane/envoy/dynamic-modules/.gitignore b/cmd/dataplane/envoy/dynamic-modules/.gitignore new file mode 100644 index 0000000000..7233fc6f32 --- /dev/null +++ b/cmd/dataplane/envoy/dynamic-modules/.gitignore @@ -0,0 +1,3 @@ +target/ +egress-policy/target/ +egress-policy-cache/target/ diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy/Cargo.lock b/cmd/dataplane/envoy/dynamic-modules/Cargo.lock similarity index 98% rename from cmd/dataplane/envoy/dynamic-modules/egress-policy/Cargo.lock rename to cmd/dataplane/envoy/dynamic-modules/Cargo.lock index d5aa3de254..bb8f846cb2 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy/Cargo.lock +++ b/cmd/dataplane/envoy/dynamic-modules/Cargo.lock @@ -343,6 +343,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "substrate-envoy-egress-policy-cache" +version = "0.1.0" +dependencies = [ + "envoy-proxy-dynamic-modules-rust-sdk", + "serde", + "serde_json", +] + [[package]] name = "syn" version = "2.0.119" diff --git a/cmd/dataplane/envoy/dynamic-modules/Cargo.toml b/cmd/dataplane/envoy/dynamic-modules/Cargo.toml new file mode 100644 index 0000000000..2a5e3f8475 --- /dev/null +++ b/cmd/dataplane/envoy/dynamic-modules/Cargo.toml @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[workspace] +resolver = "3" +members = [ + "egress-policy", + "egress-policy-cache", +] diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml new file mode 100644 index 0000000000..8f05953212 --- /dev/null +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml @@ -0,0 +1,31 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "substrate-envoy-egress-policy-cache" +version = "0.1.0" +edition = "2024" + +[lib] +name = "envoy_substrate_egress_policy_cache" +crate-type = ["cdylib"] + +[dependencies] +# Pin to v1.39.1 commit SHA +# When changing the SDK version, the version of Envoy in the cmd/dataplane/envoy/Dockerfile SHOULD match. +# SDK version SHOULD NOT be newer than Envoy's in the dataplane image, as extensions will not load if they use +# APIs not supported by the Envoy binary in the image. +envoy-proxy-dynamic-modules-rust-sdk = { git = "https://github.com/envoyproxy/envoy", rev = "b579d07d3ad7ee11d32b105e91a5a39ad24718d7" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs new file mode 100644 index 0000000000..76f626768b --- /dev/null +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -0,0 +1,175 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The egress-policy-cache HTTP filter dynamic module. + +use envoy_proxy_dynamic_modules_rust_sdk::{ + abi::{ + envoy_dynamic_module_type_on_http_filter_request_headers_status, + envoy_dynamic_module_type_on_http_filter_response_headers_status, + }, + declare_init_functions, envoy_log_error, EnvoyHttpFilter, + EnvoyHttpFilterConfig, HttpFilter, HttpFilterConfig, +}; +use serde::Deserialize; +use std::time::Duration; + +declare_init_functions!(init, new_http_filter_config_fn); + +/// Called when the dynamic module is loaded into Envoy. +fn init() -> bool { + true +} + +/// Called when a new HTTP filter configuration is created. +fn new_http_filter_config_fn( + _envoy_filter_config: &mut EC, + _filter_name: &str, + filter_config: &[u8], +) -> Option>> { + let config = parse_config(filter_config)?; + Some(Box::new(EgressPolicyCacheFilterConfig { config })) +} + +fn parse_config(filter_config: &[u8]) -> Option { + let raw = std::str::from_utf8(filter_config).ok()?; + let json_str = if raw.trim().is_empty() { "{}" } else { raw }; + serde_json::from_str(json_str) + .map_err(|e| { + envoy_log_error!("egress_policy_cache: invalid filter config: {}", e); + }) + .ok() +} + +/// Configuration for the egress-policy-cache HTTP filter. +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct Config { + #[serde(default = "default_cache_ttl", deserialize_with = "deserialize_secs")] + pub cache_ttl: Duration, + #[serde(default = "default_cache_enabled")] + pub cache_enabled: bool, +} + +fn default_cache_ttl() -> Duration { + Duration::from_secs(5) +} + +fn default_cache_enabled() -> bool { + true +} + +fn deserialize_secs<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(secs)) +} + +impl Default for Config { + fn default() -> Self { + Self { + cache_ttl: default_cache_ttl(), + cache_enabled: default_cache_enabled(), + } + } +} + +/// Per-filter-chain configuration for the egress policy cache filter. +pub struct EgressPolicyCacheFilterConfig { + pub config: Config, +} + +impl HttpFilterConfig for EgressPolicyCacheFilterConfig { + fn new_http_filter(&self, _envoy: &mut EHF) -> Box> { + Box::new(EgressPolicyCacheFilter { + config: self.config.clone(), + }) + } +} + +/// Per-stream HTTP filter instance for caching egress policy decisions. +pub struct EgressPolicyCacheFilter { + pub config: Config, +} + +impl HttpFilter for EgressPolicyCacheFilter { + fn on_request_headers( + &mut self, + _envoy_filter: &mut EHF, + _end_of_stream: bool, + ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { + let _ = (&self.config.cache_ttl, self.config.cache_enabled); + envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue + } + + fn on_response_headers( + &mut self, + _envoy_filter: &mut EHF, + _end_of_stream: bool, + ) -> envoy_dynamic_module_type_on_http_filter_response_headers_status { + let _ = (&self.config.cache_ttl, self.config.cache_enabled); + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + } +} + +#[cfg(test)] +mod tests { + use super::*; + use envoy_proxy_dynamic_modules_rust_sdk::MockEnvoyHttpFilter; + + #[test] + fn test_config_defaults_and_overrides() { + assert_eq!( + parse_config(b""), + Some(Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: true, + }) + ); + assert_eq!( + parse_config(b"{}"), + Some(Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: true, + }) + ); + assert_eq!( + parse_config(br#"{"cache_ttl": 10, "cache_enabled": false}"#), + Some(Config { + cache_ttl: Duration::from_secs(10), + cache_enabled: false, + }) + ); + assert_eq!(parse_config(b"not-json"), None); + } + + #[test] + fn test_headers_continue() { + let config = EgressPolicyCacheFilterConfig { + config: Config::default(), + }; + let mut mock_filter = MockEnvoyHttpFilter::new(); + let mut filter = config.new_http_filter(&mut mock_filter); + + assert_eq!( + filter.on_request_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue + ); + assert_eq!( + filter.on_response_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + ); + } +} diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy/.gitignore b/cmd/dataplane/envoy/dynamic-modules/egress-policy/.gitignore deleted file mode 100644 index 2f7896d1d1..0000000000 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index e2bd351a38..671a9ba3fe 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -25,7 +25,9 @@ import ( "github.com/agent-substrate/substrate/internal/ateclient" "github.com/agent-substrate/substrate/internal/portforward" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) const ( @@ -35,6 +37,15 @@ const ( // AgentGateway exposes native Prometheus metrics; it does not export these // instruments through the OTLP collector. agentGatewayRouterStatsPort = 15020 + // The egress Envoy exposes its Prometheus stats on the admin listener. + egressEnvoyAdminPort = 15000 + + // EgressExtProcIdentityStatPrefix is the Envoy ext_proc filter stat_prefix on + // the outer CONNECT leg in atenet-egress. + EgressExtProcIdentityStatPrefix = "egress_identity" + // EgressExtProcPolicyMITMStatPrefix is the Envoy ext_proc filter stat_prefix + // on the inner MITM HTTP leg in atenet-egress. + EgressExtProcPolicyMITMStatPrefix = "egress_policy_mitm" ) // PlatformMetricPrefixes are the Prometheus metric-name prefixes (OTLP dots @@ -128,6 +139,105 @@ func ScrapeCollectorMetrics(ctx context.Context) (string, error) { return string(body), nil } +// ScrapeEgressEnvoyMetrics port-forwards each ready atenet-egress pod's Envoy +// admin port and reads its /stats/prometheus endpoint, returning the +// concatenated exposition text. +func ScrapeEgressEnvoyMetrics(ctx context.Context) (string, error) { + config, err := ateclient.LoadKubeConfig(KubeConfig, KubeContext) + if err != nil { + return "", fmt.Errorf("loading kubeconfig: %w", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return "", fmt.Errorf("creating k8s client: %w", err) + } + + ns := SystemNamespace() + pods, err := clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ + LabelSelector: "app=atenet-egress", + }) + if err != nil { + return "", fmt.Errorf("listing atenet-egress pods: %w", err) + } + + var b strings.Builder + ready := 0 + for i := range pods.Items { + pod := &pods.Items[i] + if !portforward.IsPodReady(pod) { + continue + } + ready++ + body, err := scrapeEgressEnvoyPod(ctx, config, clientset, ns, pod.Name) + if err != nil { + return "", err + } + b.WriteString(body) + } + if ready == 0 { + return "", fmt.Errorf("no ready atenet-egress pods in %s", ns) + } + return b.String(), nil +} + +func scrapeEgressEnvoyPod(ctx context.Context, config *rest.Config, clientset kubernetes.Interface, namespace, podName string) (string, error) { + localPort, stop, err := portforward.PodPortForward(ctx, config, clientset, namespace, podName, egressEnvoyAdminPort) + if err != nil { + return "", err + } + defer stop() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/stats/prometheus", localPort), nil) + if err != nil { + return "", err + } + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return "", fmt.Errorf("scraping egress Envoy metrics on %s: %w", podName, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading egress Envoy metrics on %s: %w", podName, err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("egress Envoy metrics on %s returned %d: %s", podName, resp.StatusCode, body) + } + return string(body), nil +} + +// EgressExtProcStreamCounts returns the envoy_http_ext_proc__streams_started +// count for each egress ext_proc stat_prefix in an Envoy /stats/prometheus +// scrape, summed across instances. +func EgressExtProcStreamCounts(scrape string) map[string]int { + counts := map[string]int{} + for _, line := range strings.Split(scrape, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name := metricNameFromLine(line) + rest, ok := strings.CutPrefix(name, "envoy_http_ext_proc_") + if !ok { + continue + } + statPrefix, ok := strings.CutSuffix(rest, "_streams_started") + if !ok { + statPrefix, ok = strings.CutSuffix(rest, "_streams_started_total") + } + if !ok || statPrefix == "" { + continue + } + fields := strings.Fields(line) + v, err := strconv.ParseFloat(fields[len(fields)-1], 64) + if err != nil { + continue + } + counts[statPrefix] += int(v) + } + return counts +} + // MissingPlatformMetrics returns the prefixes with no matching series in the // Prometheus exposition text. A metric matches when its name equals a prefix or // begins with prefix+"_"; the underscore boundary stops "ate_actor_restore" from diff --git a/internal/e2e/collector_metrics_test.go b/internal/e2e/collector_metrics_test.go index b0efb72843..06cc8c5c08 100644 --- a/internal/e2e/collector_metrics_test.go +++ b/internal/e2e/collector_metrics_test.go @@ -257,3 +257,58 @@ func TestStatesNotAdvanced(t *testing.T) { }) } } + +const sampleEgressEnvoyScrape = `# TYPE envoy_http_ext_proc_egress_identity_streams_started counter +envoy_http_ext_proc_egress_identity_streams_started{envoy_http_conn_manager_prefix="egress_connect"} 6 +# TYPE envoy_http_ext_proc_egress_policy_mitm_streams_started counter +envoy_http_ext_proc_egress_policy_mitm_streams_started{envoy_http_conn_manager_prefix="mitm_http"} 1 +# TYPE envoy_http_ext_proc_egress_policy_cleartext_streams_started counter +envoy_http_ext_proc_egress_policy_cleartext_streams_started{envoy_http_conn_manager_prefix="mitm_cleartext"} 0 +# TYPE envoy_http_ext_proc_egress_identity_streams_closed counter +envoy_http_ext_proc_egress_identity_streams_closed{envoy_http_conn_manager_prefix="egress_connect"} 6 +` + +func TestEgressExtProcStreamCounts(t *testing.T) { + tests := []struct { + name string + scrape string + want map[string]int + }{ + { + name: "per-leg ext_proc counters", + scrape: sampleEgressEnvoyScrape, + want: map[string]int{ + EgressExtProcIdentityStatPrefix: 6, + EgressExtProcPolicyMITMStatPrefix: 1, + "egress_policy_cleartext": 0, + }, + }, + { + name: "summed across replicas", + scrape: sampleEgressEnvoyScrape + sampleEgressEnvoyScrape, + want: map[string]int{ + EgressExtProcIdentityStatPrefix: 12, + EgressExtProcPolicyMITMStatPrefix: 2, + "egress_policy_cleartext": 0, + }, + }, + { + name: "empty scrape", + scrape: "", + want: map[string]int{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := EgressExtProcStreamCounts(tt.scrape) + if len(got) != len(tt.want) { + t.Fatalf("EgressExtProcStreamCounts() = %v, want %v", got, tt.want) + } + for prefix, want := range tt.want { + if got[prefix] != want { + t.Errorf("EgressExtProcStreamCounts()[%q] = %d, want %d", prefix, got[prefix], want) + } + } + }) + } +} diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 4e73588f12..0dafacf736 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -99,8 +99,17 @@ func TestActorEgressMITMTrust(t *testing.T) { // deleted it), leaving the gateway briefly signing with the old CA — so // certificate failures retry for one propagation window before counting. deadline := time.Now().Add(2 * time.Minute) - var pos fetchResponse + var ( + beforeExtProc map[string]int + pos fetchResponse + ) for { + beforeScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) + } + beforeExtProc = e2e.EgressExtProcStreamCounts(beforeScrape) + pos = probeFetch(t, ctx, rc, id, origin, "bundle") isCertErr := strings.Contains(pos.Error, "certificate") || strings.Contains(pos.Error, "x509") if pos.Error == "" || !isCertErr || time.Now().After(deadline) { @@ -168,6 +177,26 @@ func TestActorEgressMITMTrust(t *testing.T) { case denied.Status != "": t.Errorf("fetch of a host outside the policy got status %s with error %q, want no HTTP exchange at all", denied.Status, denied.Error) } + + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) + } + afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) + + // Each of the 6 fetches opens a CONNECT tunnel (egress_identity), while + // only the positive MITM fetch completes the inner TLS handshake and sends + // an HTTP request through the MITM chain (egress_policy_mitm). + const ( + wantIdentityCalls = 6 + wantMITMPolicyCalls = 1 + ) + if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantIdentityCalls { + t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcIdentityStatPrefix, got, wantIdentityCalls) + } + if got := afterExtProc[e2e.EgressExtProcPolicyMITMStatPrefix] - beforeExtProc[e2e.EgressExtProcPolicyMITMStatPrefix]; got != wantMITMPolicyCalls { + t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcPolicyMITMStatPrefix, got, wantMITMPolicyCalls) + } } // egressOriginHost is the one host the probe actor's EgressPolicy allows. diff --git a/internal/portforward/portforward.go b/internal/portforward/portforward.go index 7fa750b251..4b79b327a8 100644 --- a/internal/portforward/portforward.go +++ b/internal/portforward/portforward.go @@ -48,7 +48,7 @@ func ServicePortForward(ctx context.Context, config *rest.Config, clientset kube if err != nil { return 0, nil, err } - return podPortForward(ctx, config, clientset, namespace, pod.Name, targetPort) + return PodPortForward(ctx, config, clientset, namespace, pod.Name, targetPort) } func firstReadyPodForService(ctx context.Context, clientset kubernetes.Interface, namespace, service string) (*corev1.Pod, *corev1.Service, error) { @@ -102,7 +102,10 @@ func resolveTargetPort(svc *corev1.Service, pod *corev1.Pod, servicePort int32) return 0, fmt.Errorf("service %s has no port %d", svc.Name, servicePort) } -func podPortForward(ctx context.Context, config *rest.Config, clientset kubernetes.Interface, namespace, podName string, targetPort int32) (int, func(), error) { +// PodPortForward forwards a random local port to targetPort of the named pod. +// It returns the chosen local port and a stop func the caller must invoke to +// tear the tunnel down. +func PodPortForward(ctx context.Context, config *rest.Config, clientset kubernetes.Interface, namespace, podName string, targetPort int32) (int, func(), error) { req := clientset.CoreV1().RESTClient().Post(). Resource("pods"). Namespace(namespace). From 335fc424959d8b24f899ffc2982d9841d7caef84 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Sun, 4 Oct 2026 02:03:33 +0000 Subject: [PATCH 02/23] WiP Signed-off-by: Yan Avlasov --- .../envoy/dynamic-modules/Cargo.lock | 65 ++++- .../egress-policy-cache/Cargo.toml | 2 + .../egress-policy-cache/src/lib.rs | 247 +++++++++++++++++- manifests/ate-install/atenet-egress.yaml | 6 + 4 files changed, 303 insertions(+), 17 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/Cargo.lock b/cmd/dataplane/envoy/dynamic-modules/Cargo.lock index bb8f846cb2..61f3b30498 100644 --- a/cmd/dataplane/envoy/dynamic-modules/Cargo.lock +++ b/cmd/dataplane/envoy/dynamic-modules/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstyle" version = "1.0.14" @@ -39,9 +45,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "cexpr" @@ -54,9 +60,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "clang-sys" @@ -90,6 +96,18 @@ dependencies = [ "mockall", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "fragile" version = "2.1.0" @@ -111,6 +129,17 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "itertools" version = "0.13.0" @@ -128,9 +157,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libloading" @@ -148,6 +177,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown", +] + [[package]] name = "memchr" version = "2.8.3" @@ -348,8 +386,10 @@ name = "substrate-envoy-egress-policy-cache" version = "0.1.0" dependencies = [ "envoy-proxy-dynamic-modules-rust-sdk", + "lru", "serde", "serde_json", + "thread_local", ] [[package]] @@ -380,11 +420,20 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "windows-link" diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml index 8f05953212..0e78356930 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/Cargo.toml @@ -27,5 +27,7 @@ crate-type = ["cdylib"] # SDK version SHOULD NOT be newer than Envoy's in the dataplane image, as extensions will not load if they use # APIs not supported by the Envoy binary in the image. envoy-proxy-dynamic-modules-rust-sdk = { git = "https://github.com/envoyproxy/envoy", rev = "b579d07d3ad7ee11d32b105e91a5a39ad24718d7" } +lru = "0.16" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +thread_local = "1.1" diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 76f626768b..17c9817627 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -16,14 +16,26 @@ use envoy_proxy_dynamic_modules_rust_sdk::{ abi::{ + envoy_dynamic_module_type_attribute_id, envoy_dynamic_module_type_on_http_filter_request_headers_status, envoy_dynamic_module_type_on_http_filter_response_headers_status, }, declare_init_functions, envoy_log_error, EnvoyHttpFilter, EnvoyHttpFilterConfig, HttpFilter, HttpFilterConfig, }; +use lru::LruCache; use serde::Deserialize; +use std::cell::RefCell; +use std::num::NonZeroUsize; +use std::sync::Arc; use std::time::Duration; +use thread_local::ThreadLocal; + +/// Key holding the egress policy SNI rules JSON. +pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; + +/// Thread-local store of LRU caches with `String` keys and values. +pub type ThreadLocalCache = ThreadLocal>>; declare_init_functions!(init, new_http_filter_config_fn); @@ -39,7 +51,7 @@ fn new_http_filter_config_fn( filter_config: &[u8], ) -> Option>> { let config = parse_config(filter_config)?; - Some(Box::new(EgressPolicyCacheFilterConfig { config })) + Some(Box::new(EgressPolicyCacheFilterConfig::new(config))) } fn parse_config(filter_config: &[u8]) -> Option { @@ -59,6 +71,8 @@ pub struct Config { pub cache_ttl: Duration, #[serde(default = "default_cache_enabled")] pub cache_enabled: bool, + #[serde(default = "default_max_cache_items", alias = "max-cache-items")] + pub max_cache_items: usize, } fn default_cache_ttl() -> Duration { @@ -69,6 +83,10 @@ fn default_cache_enabled() -> bool { true } +fn default_max_cache_items() -> usize { + 1000 +} + fn deserialize_secs<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -82,19 +100,43 @@ impl Default for Config { Self { cache_ttl: default_cache_ttl(), cache_enabled: default_cache_enabled(), + max_cache_items: default_max_cache_items(), } } } +fn new_lru_cache(max_cache_items: usize) -> RefCell> { + let cap = NonZeroUsize::new(max_cache_items).unwrap_or(NonZeroUsize::MIN); + RefCell::new(LruCache::new(cap)) +} + /// Per-filter-chain configuration for the egress policy cache filter. pub struct EgressPolicyCacheFilterConfig { pub config: Config, + pub cache: Arc, +} + +impl EgressPolicyCacheFilterConfig { + pub fn new(config: Config) -> Self { + Self { + config, + cache: Arc::new(ThreadLocal::new()), + } + } + + /// Returns a reference to the current thread's LRU cache, initializing it + /// with capacity `config.max_cache_items` on first access. + pub fn local_cache(&self) -> &RefCell> { + let max_items = self.config.max_cache_items; + self.cache.get_or(|| new_lru_cache(max_items)) + } } impl HttpFilterConfig for EgressPolicyCacheFilterConfig { fn new_http_filter(&self, _envoy: &mut EHF) -> Box> { Box::new(EgressPolicyCacheFilter { config: self.config.clone(), + cache: Arc::clone(&self.cache), }) } } @@ -102,6 +144,43 @@ impl HttpFilterConfig for EgressPolicyCacheFilterConf /// Per-stream HTTP filter instance for caching egress policy decisions. pub struct EgressPolicyCacheFilter { pub config: Config, + pub cache: Arc, +} + +impl EgressPolicyCacheFilter { + /// Returns a reference to the current thread's LRU cache, initializing it + /// with capacity `config.max_cache_items` on first access. + pub fn local_cache(&self) -> &RefCell> { + let max_items = self.config.max_cache_items; + self.cache.get_or(|| new_lru_cache(max_items)) + } +} + +fn is_http_200(envoy_filter: &EHF) -> bool { + if let Some(status) = envoy_filter.get_response_header_value(":status") { + return status.as_slice() == b"200"; + } + false +} + +fn read_egress_policy(envoy_filter: &EHF) -> Option { + if let Some(buf) = envoy_filter.get_filter_state_bytes(ATE_POLICY_EGRESS.as_bytes()) + && let Ok(s) = std::str::from_utf8(buf.as_slice()) + && !s.is_empty() { + return Some(s.to_owned()); + } + None +} + +fn read_peer_cert_digest(envoy_filter: &EHF) -> Option { + let buf = envoy_filter.get_attribute_string( + envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest, + )?; + let s = std::str::from_utf8(buf.as_slice()).ok()?; + if s.is_empty() { + return None; + } + Some(s.to_owned()) } impl HttpFilter for EgressPolicyCacheFilter { @@ -110,16 +189,27 @@ impl HttpFilter for EgressPolicyCacheFilter { _envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let _ = (&self.config.cache_ttl, self.config.cache_enabled); + let _ = ( + &self.config.cache_ttl, + self.config.cache_enabled, + self.local_cache(), + ); envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue } fn on_response_headers( &mut self, - _envoy_filter: &mut EHF, + envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_response_headers_status { - let _ = (&self.config.cache_ttl, self.config.cache_enabled); + // Store policy in cache in case it was authorized (200 response) + // Actor identity comes from cert's subject. Use cert digest as key, since + // it uniquely identifies the cert and hence actor. + if is_http_200(envoy_filter) + && let Some(policy) = read_egress_policy(envoy_filter) + && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { + self.local_cache().borrow_mut().put(cert_digest, policy); + } envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue } } @@ -127,7 +217,7 @@ impl HttpFilter for EgressPolicyCacheFilter { #[cfg(test)] mod tests { use super::*; - use envoy_proxy_dynamic_modules_rust_sdk::MockEnvoyHttpFilter; + use envoy_proxy_dynamic_modules_rust_sdk::{EnvoyBuffer, MockEnvoyHttpFilter}; #[test] fn test_config_defaults_and_overrides() { @@ -136,6 +226,7 @@ mod tests { Some(Config { cache_ttl: Duration::from_secs(5), cache_enabled: true, + max_cache_items: 1000, }) ); assert_eq!( @@ -143,13 +234,25 @@ mod tests { Some(Config { cache_ttl: Duration::from_secs(5), cache_enabled: true, + max_cache_items: 1000, }) ); assert_eq!( - parse_config(br#"{"cache_ttl": 10, "cache_enabled": false}"#), + parse_config( + br#"{"cache_ttl": 10, "cache_enabled": false, "max_cache_items": 500}"# + ), Some(Config { cache_ttl: Duration::from_secs(10), cache_enabled: false, + max_cache_items: 500, + }) + ); + assert_eq!( + parse_config(br#"{"max-cache-items": 250}"#), + Some(Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: true, + max_cache_items: 250, }) ); assert_eq!(parse_config(b"not-json"), None); @@ -157,10 +260,16 @@ mod tests { #[test] fn test_headers_continue() { - let config = EgressPolicyCacheFilterConfig { - config: Config::default(), - }; + let config = EgressPolicyCacheFilterConfig::new(Config::default()); let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_response_header_value() + .withf(|key| key == ":status") + .returning(|_| None); + mock_filter + .expect_get_attribute_int() + .withf(|id| *id == envoy_dynamic_module_type_attribute_id::ResponseCode) + .return_const(None); let mut filter = config.new_http_filter(&mut mock_filter); assert_eq!( @@ -172,4 +281,124 @@ mod tests { envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue ); } + + #[test] + fn test_on_response_headers_caches_on_200_with_policy_and_cert_digest() { + let config = EgressPolicyCacheFilterConfig::new(Config::default()); + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_response_header_value() + .withf(|key| key == ":status") + .returning(|_| Some(EnvoyBuffer::new(b"200"))); + mock_filter + .expect_get_filter_state_bytes() + .withf(|key| key == ATE_POLICY_EGRESS.as_bytes()) + .returning(|_| { + Some(EnvoyBuffer::new( + br#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#, + )) + }); + mock_filter + .expect_get_attribute_string() + .withf(|id| { + *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest + }) + .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_response_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + ); + + assert_eq!( + config + .local_cache() + .borrow_mut() + .get("abc123digest") + .map(String::as_str), + Some(r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#) + ); + } + + #[test] + fn test_on_response_headers_skips_cache_on_non_200() { + let config = EgressPolicyCacheFilterConfig::new(Config::default()); + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_response_header_value() + .withf(|key| key == ":status") + .returning(|_| Some(EnvoyBuffer::new(b"403"))); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_response_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + ); + assert_eq!(config.local_cache().borrow().len(), 0); + } + + #[test] + fn test_thread_local_lru_cache_shared_across_filters_and_isolated_across_threads() { + let filter_config = Arc::new(EgressPolicyCacheFilterConfig::new(Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: true, + max_cache_items: 2, + })); + + let filter1 = EgressPolicyCacheFilter { + config: filter_config.config.clone(), + cache: Arc::clone(&filter_config.cache), + }; + let filter2 = EgressPolicyCacheFilter { + config: filter_config.config.clone(), + cache: Arc::clone(&filter_config.cache), + }; + + assert_eq!(filter1.local_cache().borrow().cap().get(), 2); + + filter1 + .local_cache() + .borrow_mut() + .put("k1".to_string(), "v1".to_string()); + filter1 + .local_cache() + .borrow_mut() + .put("k2".to_string(), "v2".to_string()); + + // Second filter created from the same config on the same thread sees the entries. + assert_eq!( + filter2.local_cache().borrow_mut().get("k1").map(String::as_str), + Some("v1") + ); + + // Inserting a 3rd item evicts the least-recently-used item ("k2"). + filter2 + .local_cache() + .borrow_mut() + .put("k3".to_string(), "v3".to_string()); + assert_eq!(filter1.local_cache().borrow_mut().get("k2"), None); + assert_eq!( + filter1.local_cache().borrow_mut().get("k1").map(String::as_str), + Some("v1") + ); + assert_eq!( + filter1.local_cache().borrow_mut().get("k3").map(String::as_str), + Some("v3") + ); + + // A filter on another worker thread gets its own empty thread-local cache. + let filter_config_clone = Arc::clone(&filter_config); + let other_thread_len = std::thread::spawn(move || { + let other_filter = EgressPolicyCacheFilter { + config: filter_config_clone.config.clone(), + cache: Arc::clone(&filter_config_clone.cache), + }; + assert_eq!(other_filter.local_cache().borrow().cap().get(), 2); + other_filter.local_cache().borrow().len() + }) + .join() + .unwrap(); + assert_eq!(other_thread_len, 0); + } } diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 4974422972..3c39df99ce 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -186,6 +186,12 @@ data: text_format_source: inline_string: "%DOWNSTREAM_PEER_URI_SAN%" omit_empty_values: true + - name: envoy.filters.http.dynamic_modules + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_modules.v3.DynamicModuleFilter + dynamic_module_config: + name: envoy_substrate_egress_policy_cache + filter_name: envoy_substrate_egress_policy_cache # The CONNECT leg. The sidecar authenticates the actor from its # certificate (see cmd/atenet/internal/router/egress) and returns # the SNI rules in dev.ate.policy.egress and the dialed port in From a06555514d22d0ad62fb0fb536fd812866f0f142 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Sun, 4 Oct 2026 18:34:12 +0000 Subject: [PATCH 03/23] WiP Signed-off-by: Yan Avlasov --- .github/workflows/pr-workflow.yaml | 4 + cmd/ate-setup/internal/config/config.go | 4 + cmd/ate-setup/internal/steps/overlay.go | 20 ++ cmd/ate-setup/internal/steps/overlay_test.go | 21 +- .../egress-policy-cache/src/lib.rs | 304 +++++++++++++++++- demos/egress/main.go | 8 +- demos/egress/main_test.go | 25 ++ internal/e2e/collector_metrics.go | 33 ++ internal/e2e/collector_metrics_test.go | 40 +++ .../suites/networking/egresspolicy_test.go | 39 +++ manifests/ate-install/atenet-egress.yaml | 2 + 11 files changed, 483 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index ad6008379f..9a32bc11df 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -150,6 +150,7 @@ jobs: ) }} env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} + E2E_ENVOY_CONCURRENCY: "1" # The installer requires a credential provider selection, and deploys the # bundled Kubernetes Secrets provider with the control plane. Envoy only: # it has no agentgateway credential-provider configuration yet. Job-wide @@ -190,6 +191,8 @@ jobs: - name: Install Agent Substrate (${{ matrix.dataplane }}) # The dataplane selection applies to both the ingress router and egress # gateway. + env: + E2E_ENVOY_CONCURRENCY: "1" run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. @@ -204,6 +207,7 @@ jobs: # redeploys the control plane, so retain the selected dataplane. env: ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} + E2E_ENVOY_CONCURRENCY: "1" run: hack/run-microvm-demo-kind.sh - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index 96b6af4130..6298662e14 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -198,6 +198,9 @@ type Config struct { // BenchmarkActorMemory is the memory limit for benchmark actors // (BENCHMARK_ACTOR_MEMORY). Empty leaves the workload default in place. BenchmarkActorMemory string + // EnvoyConcurrency overrides the Envoy egress gateway worker thread count + // (E2E_ENVOY_CONCURRENCY). Empty leaves Envoy's default concurrency in place. + EnvoyConcurrency string // kubeconfigEnv is what ScriptEnv exports as $KUBECONFIG. Unlike Kubeconfig // it may be a PATH-style list of files, which kubectl understands and @@ -375,6 +378,7 @@ func Load(opts Options) (*Config, error) { AnthropicAPIKey: env["ANTHROPIC_API_KEY"], OtlpEndpoint: firstNonEmpty(opts.OtlpEndpoint, env["ATE_OTLP_ENDPOINT"]), BenchmarkActorMemory: env["BENCHMARK_ACTOR_MEMORY"], + EnvoyConcurrency: env["E2E_ENVOY_CONCURRENCY"], kubeconfigEnv: kubeconfigEnv, shellEnv: env, } diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 49388870a6..28ac34ef20 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -168,6 +168,7 @@ func (e *Env) renderAtenetEgressManifest(ctx context.Context, provider config.Cr return nil, err } raw = e.patchEnvoyDataplaneImage(raw, imageReference) + raw = e.patchEnvoyConcurrency(raw, e.Cfg.EnvoyConcurrency) rendered, err := e.renderBytes(raw) if err != nil { return nil, err @@ -181,6 +182,25 @@ func (e *Env) patchEnvoyDataplaneImage(raw []byte, imageRef string) []byte { return bytes.ReplaceAll(raw, []byte("${ENVOY_DATAPLANE_IMAGE}"), []byte(imageRef)) } +// patchEnvoyConcurrency replaces the - ${E2E_ENVOY_CONCURRENCY} placeholder in +// the manifest with --concurrency when concurrency is set, or removes +// the placeholder line when empty. +func (e *Env) patchEnvoyConcurrency(raw []byte, concurrency string) []byte { + const placeholder = "- ${E2E_ENVOY_CONCURRENCY}" + var out []string + for _, line := range strings.Split(string(raw), "\n") { + if strings.TrimSpace(line) == placeholder { + if concurrency != "" { + indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + out = append(out, indent+"- --concurrency", fmt.Sprintf("%s- %q", indent, concurrency)) + } + continue + } + out = append(out, line) + } + return []byte(strings.Join(out, "\n")) +} + // patchAtenetEgressInject replaces the #ATE_EGRESS_INJECT_FLAGS marker in the // egress sidecar's args with the credential-provider flags, or removes it when // injection is off, which leaves the gateway with no provider. It takes the diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index f982083d7f..19e787d602 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -688,7 +688,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { if !strings.Contains(string(out), "image: "+want) { t.Errorf("rendered manifest does not install %s:\n%s", want, out) } - for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "ko://"} { + for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "${E2E_ENVOY_CONCURRENCY}", "ko://"} { if strings.Contains(string(out), leftover) { t.Errorf("rendered manifest still contains %q", leftover) } @@ -733,3 +733,22 @@ func TestPatchEnvoyDataplaneImage(t *testing.T) { t.Errorf("patchEnvoyDataplaneImage() = %q, want %q", got, want) } } + +func TestPatchEnvoyConcurrency(t *testing.T) { + e := &Env{} + raw := []byte("containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - ${E2E_ENVOY_CONCURRENCY}\n") + + t.Run("set", func(t *testing.T) { + want := "containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - --concurrency\n - \"1\"\n" + if got := string(e.patchEnvoyConcurrency(raw, "1")); got != want { + t.Errorf("patchEnvoyConcurrency(1) = %q, want %q", got, want) + } + }) + + t.Run("unset", func(t *testing.T) { + want := "containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n" + if got := string(e.patchEnvoyConcurrency(raw, "")); got != want { + t.Errorf("patchEnvoyConcurrency(\"\") = %q, want %q", got, want) + } + }) +} diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 17c9817627..33cb085380 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -20,7 +20,7 @@ use envoy_proxy_dynamic_modules_rust_sdk::{ envoy_dynamic_module_type_on_http_filter_request_headers_status, envoy_dynamic_module_type_on_http_filter_response_headers_status, }, - declare_init_functions, envoy_log_error, EnvoyHttpFilter, + declare_init_functions, envoy_log_error, EnvoyCounterId, EnvoyHttpFilter, EnvoyHttpFilterConfig, HttpFilter, HttpFilterConfig, }; use lru::LruCache; @@ -34,6 +34,12 @@ use thread_local::ThreadLocal; /// Key holding the egress policy SNI rules JSON. pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; +/// Counter name for egress policy cache hits on CONNECT. +pub const CONNECT_CACHE_HIT_COUNTER: &str = "ate_egress.connect_cache_hit"; + +/// Counter name for egress policy cache misses on CONNECT. +pub const CONNECT_CACHE_MISS_COUNTER: &str = "ate_egress.connect_cache_miss"; + /// Thread-local store of LRU caches with `String` keys and values. pub type ThreadLocalCache = ThreadLocal>>; @@ -46,12 +52,22 @@ fn init() -> bool { /// Called when a new HTTP filter configuration is created. fn new_http_filter_config_fn( - _envoy_filter_config: &mut EC, + envoy_filter_config: &mut EC, _filter_name: &str, filter_config: &[u8], ) -> Option>> { let config = parse_config(filter_config)?; - Some(Box::new(EgressPolicyCacheFilterConfig::new(config))) + let cache_hit_counter = envoy_filter_config + .define_counter(CONNECT_CACHE_HIT_COUNTER) + .ok()?; + let cache_miss_counter = envoy_filter_config + .define_counter(CONNECT_CACHE_MISS_COUNTER) + .ok()?; + Some(Box::new(EgressPolicyCacheFilterConfig::with_counters( + config, + cache_hit_counter, + cache_miss_counter, + ))) } fn parse_config(filter_config: &[u8]) -> Option { @@ -114,13 +130,25 @@ fn new_lru_cache(max_cache_items: usize) -> RefCell> { pub struct EgressPolicyCacheFilterConfig { pub config: Config, pub cache: Arc, + pub cache_hit_counter: EnvoyCounterId, + pub cache_miss_counter: EnvoyCounterId, } impl EgressPolicyCacheFilterConfig { pub fn new(config: Config) -> Self { + Self::with_counters(config, EnvoyCounterId(0), EnvoyCounterId(1)) + } + + pub fn with_counters( + config: Config, + cache_hit_counter: EnvoyCounterId, + cache_miss_counter: EnvoyCounterId, + ) -> Self { Self { config, cache: Arc::new(ThreadLocal::new()), + cache_hit_counter, + cache_miss_counter, } } @@ -137,6 +165,8 @@ impl HttpFilterConfig for EgressPolicyCacheFilterConf Box::new(EgressPolicyCacheFilter { config: self.config.clone(), cache: Arc::clone(&self.cache), + cache_hit_counter: self.cache_hit_counter, + cache_miss_counter: self.cache_miss_counter, }) } } @@ -145,6 +175,8 @@ impl HttpFilterConfig for EgressPolicyCacheFilterConf pub struct EgressPolicyCacheFilter { pub config: Config, pub cache: Arc, + pub cache_hit_counter: EnvoyCounterId, + pub cache_miss_counter: EnvoyCounterId, } impl EgressPolicyCacheFilter { @@ -186,14 +218,18 @@ fn read_peer_cert_digest(envoy_filter: &EHF) -> Option HttpFilter for EgressPolicyCacheFilter { fn on_request_headers( &mut self, - _envoy_filter: &mut EHF, + envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let _ = ( - &self.config.cache_ttl, - self.config.cache_enabled, - self.local_cache(), - ); + let _ = (&self.config.cache_ttl, self.config.cache_enabled); + if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) + && let Some(policy) = self.local_cache().borrow_mut().get(&cert_digest).cloned() + { + envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS.as_bytes(), policy.as_bytes()); + let _ = envoy_filter.increment_counter(self.cache_hit_counter, 1); + } else { + let _ = envoy_filter.increment_counter(self.cache_miss_counter, 1); + } envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue } @@ -217,7 +253,173 @@ impl HttpFilter for EgressPolicyCacheFilter { #[cfg(test)] mod tests { use super::*; - use envoy_proxy_dynamic_modules_rust_sdk::{EnvoyBuffer, MockEnvoyHttpFilter}; + use envoy_proxy_dynamic_modules_rust_sdk::{ + abi::{ + envoy_dynamic_module_type_http_callout_init_result, + envoy_dynamic_module_type_metrics_result, + }, + EnvoyBuffer, EnvoyCounterVecId, EnvoyGaugeId, EnvoyGaugeVecId, + EnvoyHistogramId, EnvoyHistogramVecId, EnvoyHttpFilterConfigScheduler, + MockEnvoyHttpFilter, + }; + + #[derive(Default)] + struct TestEnvoyHttpFilterConfig { + defined_counters: Vec, + } + + impl EnvoyHttpFilterConfig for TestEnvoyHttpFilterConfig { + fn define_counter( + &mut self, + name: &str, + ) -> Result { + self.defined_counters.push(name.to_string()); + Ok(EnvoyCounterId(self.defined_counters.len())) + } + fn define_counter_vec( + &mut self, + _: &str, + _: &[&str], + ) -> Result { + todo!() + } + fn define_gauge( + &mut self, + _: &str, + ) -> Result { + todo!() + } + fn define_gauge_vec( + &mut self, + _: &str, + _: &[&str], + ) -> Result { + todo!() + } + fn define_histogram( + &mut self, + _: &str, + ) -> Result { + todo!() + } + fn define_histogram_vec( + &mut self, + _: &str, + _: &[&str], + ) -> Result { + todo!() + } + fn increment_counter( + &self, + _: EnvoyCounterId, + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn increment_counter_vec( + &self, + _: EnvoyCounterVecId, + _: &[&str], + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn increase_gauge( + &self, + _: EnvoyGaugeId, + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn increase_gauge_vec( + &self, + _: EnvoyGaugeVecId, + _: &[&str], + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn decrease_gauge( + &self, + _: EnvoyGaugeId, + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn decrease_gauge_vec( + &self, + _: EnvoyGaugeVecId, + _: &[&str], + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn set_gauge( + &self, + _: EnvoyGaugeId, + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn set_gauge_vec( + &self, + _: EnvoyGaugeVecId, + _: &[&str], + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn record_histogram_value( + &self, + _: EnvoyHistogramId, + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn record_histogram_value_vec( + &self, + _: EnvoyHistogramVecId, + _: &[&str], + _: u64, + ) -> Result<(), envoy_dynamic_module_type_metrics_result> { + todo!() + } + fn new_scheduler(&self) -> Box { + todo!() + } + fn send_http_callout<'a>( + &mut self, + _: &'a str, + _: &'a [(&'a str, &'a [u8])], + _: Option<&'a [u8]>, + _: u64, + ) -> (envoy_dynamic_module_type_http_callout_init_result, u64) { + todo!() + } + fn start_http_stream<'a>( + &mut self, + _: &'a str, + _: &'a [(&'a str, &'a [u8])], + _: Option<&'a [u8]>, + _: bool, + _: u64, + ) -> (envoy_dynamic_module_type_http_callout_init_result, u64) { + todo!() + } + unsafe fn send_http_stream_data(&mut self, _: u64, _: &[u8], _: bool) -> bool { + todo!() + } + unsafe fn send_http_stream_trailers<'a>( + &mut self, + _: u64, + _: &'a [(&'a str, &'a [u8])], + ) -> bool { + todo!() + } + unsafe fn reset_http_stream(&mut self, _: u64) { + todo!() + } + } #[test] fn test_config_defaults_and_overrides() { @@ -258,18 +460,45 @@ mod tests { assert_eq!(parse_config(b"not-json"), None); } + #[test] + fn test_new_http_filter_config_defines_counters() { + let mut test_config = TestEnvoyHttpFilterConfig::default(); + let filter_config = new_http_filter_config_fn::< + TestEnvoyHttpFilterConfig, + MockEnvoyHttpFilter, + >(&mut test_config, "egress_policy_cache", b"{}"); + assert!(filter_config.is_some()); + assert_eq!( + test_config.defined_counters, + vec![ + CONNECT_CACHE_HIT_COUNTER.to_string(), + CONNECT_CACHE_MISS_COUNTER.to_string(), + ] + ); + } + #[test] fn test_headers_continue() { - let config = EgressPolicyCacheFilterConfig::new(Config::default()); + let hit_id = EnvoyCounterId(1); + let miss_id = EnvoyCounterId(2); + let config = + EgressPolicyCacheFilterConfig::with_counters(Config::default(), hit_id, miss_id); let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_attribute_string() + .withf(|id| { + *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest + }) + .returning(|_| Some(EnvoyBuffer::new(b"unknown_digest"))); + mock_filter + .expect_increment_counter() + .withf(move |id, val| *id == miss_id && *val == 1) + .return_const(Ok(())) + .once(); mock_filter .expect_get_response_header_value() .withf(|key| key == ":status") .returning(|_| None); - mock_filter - .expect_get_attribute_int() - .withf(|id| *id == envoy_dynamic_module_type_attribute_id::ResponseCode) - .return_const(None); let mut filter = config.new_http_filter(&mut mock_filter); assert_eq!( @@ -282,6 +511,45 @@ mod tests { ); } + #[test] + fn test_on_request_headers_cache_hit_writes_filter_state_and_increments_hit_counter() { + let hit_id = EnvoyCounterId(1); + let miss_id = EnvoyCounterId(2); + let config = + EgressPolicyCacheFilterConfig::with_counters(Config::default(), hit_id, miss_id); + let expected_policy = r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#; + config + .local_cache() + .borrow_mut() + .put("abc123digest".to_string(), expected_policy.to_string()); + + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_attribute_string() + .withf(|id| { + *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest + }) + .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + mock_filter + .expect_set_filter_state_bytes() + .withf(move |key, val| { + key == ATE_POLICY_EGRESS.as_bytes() && val == expected_policy.as_bytes() + }) + .return_const(true) + .once(); + mock_filter + .expect_increment_counter() + .withf(move |id, val| *id == hit_id && *val == 1) + .return_const(Result::<(), envoy_dynamic_module_type_metrics_result>::Ok(())) + .once(); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_request_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue + ); + } + #[test] fn test_on_response_headers_caches_on_200_with_policy_and_cert_digest() { let config = EgressPolicyCacheFilterConfig::new(Config::default()); @@ -349,10 +617,14 @@ mod tests { let filter1 = EgressPolicyCacheFilter { config: filter_config.config.clone(), cache: Arc::clone(&filter_config.cache), + cache_hit_counter: filter_config.cache_hit_counter, + cache_miss_counter: filter_config.cache_miss_counter, }; let filter2 = EgressPolicyCacheFilter { config: filter_config.config.clone(), cache: Arc::clone(&filter_config.cache), + cache_hit_counter: filter_config.cache_hit_counter, + cache_miss_counter: filter_config.cache_miss_counter, }; assert_eq!(filter1.local_cache().borrow().cap().get(), 2); @@ -393,6 +665,8 @@ mod tests { let other_filter = EgressPolicyCacheFilter { config: filter_config_clone.config.clone(), cache: Arc::clone(&filter_config_clone.cache), + cache_hit_counter: filter_config_clone.cache_hit_counter, + cache_miss_counter: filter_config_clone.cache_miss_counter, }; assert_eq!(other_filter.local_cache().borrow().cap().get(), 2); other_filter.local_cache().borrow().len() diff --git a/demos/egress/main.go b/demos/egress/main.go index 66607e054c..936b21ff84 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -47,7 +47,8 @@ const ( ) type fetchRequest struct { - URL string `json:"url"` + URL string `json:"url"` + DisableKeepAlive bool `json:"disableKeepAlive,omitempty"` } type fetchResponse struct { @@ -133,6 +134,10 @@ func newHandler(client *http.Client) http.Handler { writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid URL: %v", err)}) return } + if input.DisableKeepAlive { + outbound.Close = true + client.CloseIdleConnections() + } if traceparent := r.Header.Get("traceparent"); traceparent != "" { outbound.Header.Set("traceparent", traceparent) } @@ -144,6 +149,7 @@ func newHandler(client *http.Client) http.Handler { defer response.Body.Close() body, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBody)) + _ = response.Body.Close() if err != nil { writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("reading response: %v", err)}) return diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index c164d3371f..6e60072dfd 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -69,6 +69,31 @@ func TestFetch(t *testing.T) { } } +func TestFetchDisableKeepAlive(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if !r.Close { + t.Errorf("upstream request Close = false, want true when disableKeepAlive is set") + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("ok")), + Header: make(http.Header), + }, nil + })} + + payload, err := json.Marshal(fetchRequest{URL: "https://allowed.example/", DisableKeepAlive: true}) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(payload))) + newHandler(client).ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK) + } +} + func TestFetchHTTPAndHTTPS(t *testing.T) { httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "from http") diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 671a9ba3fe..d7a1296ef1 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -46,6 +46,13 @@ const ( // EgressExtProcPolicyMITMStatPrefix is the Envoy ext_proc filter stat_prefix // on the inner MITM HTTP leg in atenet-egress. EgressExtProcPolicyMITMStatPrefix = "egress_policy_mitm" + + // EgressConnectCacheHitCounter is the Envoy dynamic-module counter name for + // CONNECT policy cache hits in atenet-egress. + EgressConnectCacheHitCounter = "ate_egress_connect_cache_hit" + // EgressConnectCacheMissCounter is the Envoy dynamic-module counter name for + // CONNECT policy cache misses in atenet-egress. + EgressConnectCacheMissCounter = "ate_egress_connect_cache_miss" ) // PlatformMetricPrefixes are the Prometheus metric-name prefixes (OTLP dots @@ -238,6 +245,32 @@ func EgressExtProcStreamCounts(scrape string) map[string]int { return counts } +// EgressPolicyCacheCounts returns the CONNECT policy cache hit and miss counts +// from an Envoy /stats/prometheus scrape, summed across instances. +func EgressPolicyCacheCounts(scrape string) (hits, misses int) { + for _, line := range strings.Split(scrape, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name := strings.TrimSuffix(metricNameFromLine(line), "_total") + fields := strings.Fields(line) + v, err := strconv.ParseFloat(fields[len(fields)-1], 64) + if err != nil { + continue + } + switch name { + case "envoy_dynamicmodulescustom_" + EgressConnectCacheHitCounter, + "envoy_" + EgressConnectCacheHitCounter: + hits += int(v) + case "envoy_dynamicmodulescustom_" + EgressConnectCacheMissCounter, + "envoy_" + EgressConnectCacheMissCounter: + misses += int(v) + } + } + return hits, misses +} + // MissingPlatformMetrics returns the prefixes with no matching series in the // Prometheus exposition text. A metric matches when its name equals a prefix or // begins with prefix+"_"; the underscore boundary stops "ate_actor_restore" from diff --git a/internal/e2e/collector_metrics_test.go b/internal/e2e/collector_metrics_test.go index 06cc8c5c08..8ca74923e3 100644 --- a/internal/e2e/collector_metrics_test.go +++ b/internal/e2e/collector_metrics_test.go @@ -266,6 +266,10 @@ envoy_http_ext_proc_egress_policy_mitm_streams_started{envoy_http_conn_manager_p envoy_http_ext_proc_egress_policy_cleartext_streams_started{envoy_http_conn_manager_prefix="mitm_cleartext"} 0 # TYPE envoy_http_ext_proc_egress_identity_streams_closed counter envoy_http_ext_proc_egress_identity_streams_closed{envoy_http_conn_manager_prefix="egress_connect"} 6 +# TYPE envoy_dynamicmodulescustom_ate_egress_connect_cache_hit counter +envoy_dynamicmodulescustom_ate_egress_connect_cache_hit{} 4 +# TYPE envoy_dynamicmodulescustom_ate_egress_connect_cache_miss counter +envoy_dynamicmodulescustom_ate_egress_connect_cache_miss{} 1 ` func TestEgressExtProcStreamCounts(t *testing.T) { @@ -312,3 +316,39 @@ func TestEgressExtProcStreamCounts(t *testing.T) { }) } } + +func TestEgressPolicyCacheCounts(t *testing.T) { + tests := []struct { + name string + scrape string + wantHits int + wantMisses int + }{ + { + name: "single replica", + scrape: sampleEgressEnvoyScrape, + wantHits: 4, + wantMisses: 1, + }, + { + name: "summed across replicas", + scrape: sampleEgressEnvoyScrape + sampleEgressEnvoyScrape, + wantHits: 8, + wantMisses: 2, + }, + { + name: "empty scrape", + scrape: "", + wantHits: 0, + wantMisses: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHits, gotMisses := EgressPolicyCacheCounts(tt.scrape) + if gotHits != tt.wantHits || gotMisses != tt.wantMisses { + t.Errorf("EgressPolicyCacheCounts() = (%d, %d), want (%d, %d)", gotHits, gotMisses, tt.wantHits, tt.wantMisses) + } + }) + } +} diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 32970e7958..0f3eaeb7fc 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -157,6 +157,45 @@ func TestActorEgressHTTPSByHostnamePassthrough(t *testing.T) { } } +// TestActorEgressPolicyCache sends 5 requests to example.com and checks that +// the egress policy cache counters record 4 hits and 1 miss. +func TestActorEgressPolicyCache(t *testing.T) { + ctx := context.Background() + router, actorRef := hostnamePolicyActor(t, ctx) + + beforeScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) + } + beforeHits, beforeMisses := e2e.EgressPolicyCacheCounts(beforeScrape) + + const numRequests = 5 + payload := []byte(`{"url":"https://example.com/","disableKeepAlive":true}`) + for i := range numRequests { + status, body := postThroughEgressActorUntil(t, ctx, router, actorRef, "/", payload, reached) + if status != http.StatusOK { + t.Fatalf("request %d to example.com returned HTTP %d, want 200; body: %s", i+1, status, body) + } + } + + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) + } + afterHits, afterMisses := e2e.EgressPolicyCacheCounts(afterScrape) + + const ( + wantHits = 4 + wantMisses = 1 + ) + if got := afterHits - beforeHits; got != wantHits { + t.Errorf("egress policy cache hits delta = %d, want %d", got, wantHits) + } + if got := afterMisses - beforeMisses; got != wantMisses { + t.Errorf("egress policy cache misses delta = %d, want %d", got, wantMisses) + } +} + // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller // deciding which answer is final. func fetchThroughEgressActorUntil(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, url string, done func(status int, body []byte) bool) (int, []byte) { diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 3c39df99ce..42ddb7e448 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -1214,6 +1214,8 @@ spec: - atenet-egress - --service-cluster - atenet-egress + # E2E_ENVOY_CONCURRENCY is substituted in renderAtenetEgressManifest + - ${E2E_ENVOY_CONCURRENCY} # Prevents Envoy from fast-exiting on SIGTERM before the ext-proc # sidecar finishes its drain sequence. Polls for the drain-complete # marker the sidecar writes on the shared emptyDir, terminating Envoy From 7b3bc73887054e0976bb5200e71b71a8d41cc3e3 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 10:46:44 +0000 Subject: [PATCH 04/23] Skip ext_proc Signed-off-by: Yan Avlasov --- .../router/egresspolicymanifest_test.go | 52 ++++---- .../internal/router/extproc/attributes.go | 7 ++ .../egress-policy-cache/src/lib.rs | 8 +- .../suites/networking/egresspolicy_test.go | 13 +- manifests/ate-install/atenet-egress.yaml | 114 ++++++++++++------ 5 files changed, 128 insertions(+), 66 deletions(-) diff --git a/cmd/atenet/internal/router/egresspolicymanifest_test.go b/cmd/atenet/internal/router/egresspolicymanifest_test.go index e377cbc3c8..ca92fc23f7 100644 --- a/cmd/atenet/internal/router/egresspolicymanifest_test.go +++ b/cmd/atenet/internal/router/egresspolicymanifest_test.go @@ -172,8 +172,8 @@ func TestEgressManifestsNameEveryExtProcChain(t *testing.T) { t.Run(path, func(t *testing.T) { var got []string for _, lc := range allChains(bootstrapTree(t, path)) { - h := hcm(lc.chain) - if h == nil || filterIndex(list(h, "http_filters"), extProcFilter) < 0 { + cfg, _, _ := extProcOf(lc.chain) + if cfg == nil { continue } name := str(lc.chain, "name") @@ -196,11 +196,19 @@ func TestEgressManifestsNameEveryExtProcChain(t *testing.T) { // among the http_filters, or nil and -1. func extProcOf(chain node) (node, int, []node) { filters := list(hcm(chain), "http_filters") - i := filterIndex(filters, extProcFilter) - if i < 0 { - return nil, -1, filters + if i := filterIndex(filters, extProcFilter); i >= 0 { + return child(filters[i], "typed_config"), i, filters + } + if i := filterIndex(filters, "envoy.filters.http.composite"); i >= 0 { + matchers := list(child(child(child(filters[i], "typed_config"), "matcher"), "matcher_list"), "matchers") + for _, m := range matchers { + action := child(child(child(m, "on_match"), "action"), "typed_config") + if inner := child(action, "typed_config"); str(inner, "name") == extProcFilter { + return child(inner, "typed_config"), i, filters + } + } } - return child(filters[i], "typed_config"), i, filters + return nil, -1, filters } // Every ext_proc in the egress gateway fails closed, talks to the co-located @@ -592,10 +600,9 @@ func TestEgressManifestsConnectLegHandsTheSNIRulesToTheInnerListener(t *testing. } writers := filterStateWriters(tree, extproc.EgressPolicyMetadataNamespace) - if len(writers) != 1 { - t.Fatalf("%s is set by %d filters, want exactly the CONNECT leg's copy of its answer", extproc.EgressPolicyMetadataNamespace, len(writers)) + if len(writers) != 2 { + t.Fatalf("%s is set by %d entries, want 2 (ext_proc dynamic metadata and cached filter state)", extproc.EgressPolicyMetadataNamespace, len(writers)) } - entry := writers[0] at := -1 for i, f := range filters { if str(f, "name") != setFilterStateFilter { @@ -613,18 +620,21 @@ func TestEgressManifestsConnectLegHandsTheSNIRulesToTheInnerListener(t *testing. if at < extProcAt { t.Errorf("%s is set at http_filters[%d], before ext_proc at [%d]; the metadata it reads does not exist yet", extproc.EgressPolicyMetadataNamespace, at, extProcAt) } - format := child(entry, "format_string") - if got := str(child(format, "text_format_source"), "inline_string"); got != extproc.EgressPolicyMetadataFormat { - t.Errorf("%s is set from %q, want %q", extproc.EgressPolicyMetadataNamespace, got, extproc.EgressPolicyMetadataFormat) - } - if got := str(entry, "factory_key"); got != "envoy.string" { - t.Errorf("%s uses factory %q, want envoy.string, the string accessor the module reads", extproc.EgressPolicyMetadataNamespace, got) - } - if skip, _ := entry["skip_if_empty"].(bool); !skip { - t.Errorf("%s is not skip_if_empty; an absent answer would be written as unparseable JSON", extproc.EgressPolicyMetadataNamespace) - } - if str(entry, "shared_with_upstream") == "" { - t.Errorf("%s is not shared with upstream; the inner listener would never see it", extproc.EgressPolicyMetadataNamespace) + wantFormats := []string{extproc.EgressPolicyMetadataFormat, extproc.EgressPolicyCachedFormat} + for i, entry := range writers { + format := child(entry, "format_string") + if got := str(child(format, "text_format_source"), "inline_string"); got != wantFormats[i] { + t.Errorf("%s writer[%d] is set from %q, want %q", extproc.EgressPolicyMetadataNamespace, i, got, wantFormats[i]) + } + if got := str(entry, "factory_key"); got != "envoy.string" { + t.Errorf("%s writer[%d] uses factory %q, want envoy.string, the string accessor the module reads", extproc.EgressPolicyMetadataNamespace, i, got) + } + if skip, _ := entry["skip_if_empty"].(bool); !skip { + t.Errorf("%s writer[%d] is not skip_if_empty; an absent answer would be written as unparseable JSON", extproc.EgressPolicyMetadataNamespace, i) + } + if str(entry, "shared_with_upstream") == "" { + t.Errorf("%s writer[%d] is not shared with upstream; the inner listener would never see it", extproc.EgressPolicyMetadataNamespace, i) + } } } diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index e4ac2944ca..da414e6fca 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -127,6 +127,13 @@ const UpstreamDynamicPortFilterStateKey = "envoy.upstream.dynamic_port" // EgressPolicyMetadataFormat renders EgressPolicyMetadataNamespace as JSON. const EgressPolicyMetadataFormat = "%DYNAMIC_METADATA(" + EgressPolicyMetadataNamespace + ")%" +// EgressPolicyCachedFilterStateKey holds the cached SNI rules written by the +// egress-policy-cache module before set_filter_state shares them upstream. +const EgressPolicyCachedFilterStateKey = EgressPolicyMetadataNamespace + ".cached" + +// EgressPolicyCachedFormat reads EgressPolicyCachedFilterStateKey as a plain string. +const EgressPolicyCachedFormat = "%FILTER_STATE(" + EgressPolicyCachedFilterStateKey + ":PLAIN)%" + // OriginalDstFilterStateKey is Envoy's filter-state key for the address an // ORIGINAL_DST cluster dials. The gateway never writes it and dials only by // name; the request legs read it as an attribute. diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 33cb085380..8a586438b0 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -34,6 +34,10 @@ use thread_local::ThreadLocal; /// Key holding the egress policy SNI rules JSON. pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; +/// Key holding the cached egress policy SNI rules JSON before it is copied to +/// upstream-shared filter state. +pub const ATE_POLICY_EGRESS_CACHED: &str = "dev.ate.policy.egress.cached"; + /// Counter name for egress policy cache hits on CONNECT. pub const CONNECT_CACHE_HIT_COUNTER: &str = "ate_egress.connect_cache_hit"; @@ -225,7 +229,7 @@ impl HttpFilter for EgressPolicyCacheFilter { if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) && let Some(policy) = self.local_cache().borrow_mut().get(&cert_digest).cloned() { - envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS.as_bytes(), policy.as_bytes()); + envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes(), policy.as_bytes()); let _ = envoy_filter.increment_counter(self.cache_hit_counter, 1); } else { let _ = envoy_filter.increment_counter(self.cache_miss_counter, 1); @@ -533,7 +537,7 @@ mod tests { mock_filter .expect_set_filter_state_bytes() .withf(move |key, val| { - key == ATE_POLICY_EGRESS.as_bytes() && val == expected_policy.as_bytes() + key == ATE_POLICY_EGRESS_CACHED.as_bytes() && val == expected_policy.as_bytes() }) .return_const(true) .once(); diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 0f3eaeb7fc..aa83424fa3 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -158,7 +158,8 @@ func TestActorEgressHTTPSByHostnamePassthrough(t *testing.T) { } // TestActorEgressPolicyCache sends 5 requests to example.com and checks that -// the egress policy cache counters record 4 hits and 1 miss. +// the egress policy cache counters record 4 hits and 1 miss, and that the +// outer CONNECT ext_proc filter is invoked only once. func TestActorEgressPolicyCache(t *testing.T) { ctx := context.Background() router, actorRef := hostnamePolicyActor(t, ctx) @@ -168,6 +169,7 @@ func TestActorEgressPolicyCache(t *testing.T) { t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) } beforeHits, beforeMisses := e2e.EgressPolicyCacheCounts(beforeScrape) + beforeExtProc := e2e.EgressExtProcStreamCounts(beforeScrape) const numRequests = 5 payload := []byte(`{"url":"https://example.com/","disableKeepAlive":true}`) @@ -183,10 +185,12 @@ func TestActorEgressPolicyCache(t *testing.T) { t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) } afterHits, afterMisses := e2e.EgressPolicyCacheCounts(afterScrape) + afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) const ( - wantHits = 4 - wantMisses = 1 + wantHits = 4 + wantMisses = 1 + wantExtProcCalls = 1 ) if got := afterHits - beforeHits; got != wantHits { t.Errorf("egress policy cache hits delta = %d, want %d", got, wantHits) @@ -194,6 +198,9 @@ func TestActorEgressPolicyCache(t *testing.T) { if got := afterMisses - beforeMisses; got != wantMisses { t.Errorf("egress policy cache misses delta = %d, want %d", got, wantMisses) } + if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantExtProcCalls { + t.Errorf("egress_identity ext_proc streams_started delta = %d, want %d", got, wantExtProcCalls) + } } // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 42ddb7e448..24a7a4382c 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -196,47 +196,73 @@ data: # certificate (see cmd/atenet/internal/router/egress) and returns # the SNI rules in dev.ate.policy.egress and the dialed port in # dev.ate.egress. Fails closed if the router is down. - - name: envoy.filters.http.ext_proc + # Skipped when envoy_substrate_egress_policy_cache already populated + # dev.ate.policy.egress.cached in filter state on a cache hit. + - name: envoy.filters.http.composite typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - stat_prefix: egress_identity - grpc_service: - envoy_grpc: - cluster_name: ext_proc_server - timeout: 2s - failure_mode_allow: false - # Set ext_proc timeout to 5s (from default 200ms) to handle slower ext_proc transactions. - message_timeout: 5s - # How the ext_proc server tells egress from ingress. It applies - # opposite trust models to the two directions, and dispatches on - # this Envoy-asserted filter chain name so that no client can - # select the egress path by crafting a request. The value must - # match EgressFilterChainName in - # cmd/atenet/internal/router/extproc/dispatch.go; renaming the - # filter chain above without updating it fails closed (the - # request is classified as ingress, which this --mode=egress - # instance does not serve, and is refused with a 404). - request_attributes: - - xds.filter_chain_name - processing_mode: - request_header_mode: SEND - response_header_mode: SKIP - request_body_mode: NONE - response_body_mode: NONE - request_trailer_mode: SKIP - response_trailer_mode: SKIP - # This leg authenticates and never rewrites; the sidecar may - # not touch routing or Envoy's own headers here. - mutation_rules: - disallow_system: true - disallow_is_error: true - # Envoy silently drops metadata from a namespace not listed - # here, and the filter state below would never be set. - metadata_options: - receiving_namespaces: - untyped: - - dev.ate.egress - - dev.ate.policy.egress + "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite + matcher: + matcher_list: + matchers: + - predicate: + not_matcher: + single_predicate: + input: + name: filter_state + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.FilterStateInput + key: dev.ate.policy.egress.cached + value_match: + safe_regex: + google_re2: {} + regex: ".*" + on_match: + action: + name: composite-action + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.ExecuteFilterAction + typed_config: + name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + stat_prefix: egress_identity + grpc_service: + envoy_grpc: + cluster_name: ext_proc_server + timeout: 2s + failure_mode_allow: false + # Set ext_proc timeout to 5s (from default 200ms) to handle slower ext_proc transactions. + message_timeout: 5s + # How the ext_proc server tells egress from ingress. It applies + # opposite trust models to the two directions, and dispatches on + # this Envoy-asserted filter chain name so that no client can + # select the egress path by crafting a request. The value must + # match EgressFilterChainName in + # cmd/atenet/internal/router/extproc/dispatch.go; renaming the + # filter chain above without updating it fails closed (the + # request is classified as ingress, which this --mode=egress + # instance does not serve, and is refused with a 404). + request_attributes: + - xds.filter_chain_name + processing_mode: + request_header_mode: SEND + response_header_mode: SKIP + request_body_mode: NONE + response_body_mode: NONE + request_trailer_mode: SKIP + response_trailer_mode: SKIP + # This leg authenticates and never rewrites; the sidecar may + # not touch routing or Envoy's own headers here. + mutation_rules: + disallow_system: true + disallow_is_error: true + # Envoy silently drops metadata from a namespace not listed + # here, and the filter state below would never be set. + metadata_options: + receiving_namespaces: + untyped: + - dev.ate.egress + - dev.ate.policy.egress # The CONNECT leg's answers, copied into filter state shared with # the inner listener. Must stay after ext_proc. - name: envoy.filters.http.set_filter_state @@ -264,6 +290,14 @@ data: text_format_source: inline_string: "%DYNAMIC_METADATA(dev.ate.policy.egress)%" omit_empty_values: true + - object_key: dev.ate.policy.egress + factory_key: envoy.string + skip_if_empty: true + shared_with_upstream: ONCE + format_string: + text_format_source: + inline_string: "%FILTER_STATE(dev.ate.policy.egress.cached:PLAIN)%" + omit_empty_values: true - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router From e449837fd8f0160ffb4e025df633dc66c3bc9063 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 13:43:06 +0000 Subject: [PATCH 05/23] Implement cache freshness Signed-off-by: Yan Avlasov --- .../egress-policy-cache/src/lib.rs | 208 +++++++++++++++--- .../suites/networking/egresspolicy_test.go | 55 +++++ 2 files changed, 230 insertions(+), 33 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 8a586438b0..1512b93674 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -28,7 +28,7 @@ use serde::Deserialize; use std::cell::RefCell; use std::num::NonZeroUsize; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use thread_local::ThreadLocal; /// Key holding the egress policy SNI rules JSON. @@ -44,8 +44,21 @@ pub const CONNECT_CACHE_HIT_COUNTER: &str = "ate_egress.connect_cache_hit"; /// Counter name for egress policy cache misses on CONNECT. pub const CONNECT_CACHE_MISS_COUNTER: &str = "ate_egress.connect_cache_miss"; -/// Thread-local store of LRU caches with `String` keys and values. -pub type ThreadLocalCache = ThreadLocal>>; +/// Cached egress policy value and the timestamp when it was stored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CachedPolicy { + pub policy: String, + pub stored_at: Instant, +} + +impl CachedPolicy { + pub fn new(policy: String, stored_at: Instant) -> Self { + Self { policy, stored_at } + } +} + +/// Thread-local store of LRU caches with `String` keys and [`CachedPolicy`] values. +pub type ThreadLocalCache = ThreadLocal>>; declare_init_functions!(init, new_http_filter_config_fn); @@ -125,7 +138,7 @@ impl Default for Config { } } -fn new_lru_cache(max_cache_items: usize) -> RefCell> { +fn new_lru_cache(max_cache_items: usize) -> RefCell> { let cap = NonZeroUsize::new(max_cache_items).unwrap_or(NonZeroUsize::MIN); RefCell::new(LruCache::new(cap)) } @@ -158,7 +171,7 @@ impl EgressPolicyCacheFilterConfig { /// Returns a reference to the current thread's LRU cache, initializing it /// with capacity `config.max_cache_items` on first access. - pub fn local_cache(&self) -> &RefCell> { + pub fn local_cache(&self) -> &RefCell> { let max_items = self.config.max_cache_items; self.cache.get_or(|| new_lru_cache(max_items)) } @@ -186,7 +199,7 @@ pub struct EgressPolicyCacheFilter { impl EgressPolicyCacheFilter { /// Returns a reference to the current thread's LRU cache, initializing it /// with capacity `config.max_cache_items` on first access. - pub fn local_cache(&self) -> &RefCell> { + pub fn local_cache(&self) -> &RefCell> { let max_items = self.config.max_cache_items; self.cache.get_or(|| new_lru_cache(max_items)) } @@ -199,6 +212,12 @@ fn is_http_200(envoy_filter: &EHF) -> bool { false } +fn has_cached_egress_policy(envoy_filter: &EHF) -> bool { + envoy_filter + .get_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes()) + .is_some() +} + fn read_egress_policy(envoy_filter: &EHF) -> Option { if let Some(buf) = envoy_filter.get_filter_state_bytes(ATE_POLICY_EGRESS.as_bytes()) && let Ok(s) = std::str::from_utf8(buf.as_slice()) @@ -225,10 +244,24 @@ impl HttpFilter for EgressPolicyCacheFilter { envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let _ = (&self.config.cache_ttl, self.config.cache_enabled); - if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) - && let Some(policy) = self.local_cache().borrow_mut().get(&cert_digest).cloned() - { + let _ = self.config.cache_enabled; + let cached_policy = if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { + let mut cache = self.local_cache().borrow_mut(); + if let Some(entry) = cache.get(&cert_digest) { + if entry.stored_at.elapsed() > self.config.cache_ttl { + cache.pop(&cert_digest); + None + } else { + Some(entry.policy.clone()) + } + } else { + None + } + } else { + None + }; + + if let Some(policy) = cached_policy { envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes(), policy.as_bytes()); let _ = envoy_filter.increment_counter(self.cache_hit_counter, 1); } else { @@ -242,13 +275,18 @@ impl HttpFilter for EgressPolicyCacheFilter { envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_response_headers_status { - // Store policy in cache in case it was authorized (200 response) - // Actor identity comes from cert's subject. Use cert digest as key, since - // it uniquely identifies the cert and hence actor. - if is_http_200(envoy_filter) - && let Some(policy) = read_egress_policy(envoy_filter) - && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { - self.local_cache().borrow_mut().put(cert_digest, policy); + // Store policy in cache in case it was authorized (200 response) on a + // cache miss. Actor identity comes from cert's subject. Use cert digest as + // key, since it uniquely identifies the cert and hence actor. + if is_http_200(envoy_filter) + && !has_cached_egress_policy(envoy_filter) + && let Some(policy) = read_egress_policy(envoy_filter) + && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) + { + self.local_cache().borrow_mut().put( + cert_digest, + CachedPolicy::new(policy, Instant::now()), + ); } envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue } @@ -522,10 +560,10 @@ mod tests { let config = EgressPolicyCacheFilterConfig::with_counters(Config::default(), hit_id, miss_id); let expected_policy = r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#; - config - .local_cache() - .borrow_mut() - .put("abc123digest".to_string(), expected_policy.to_string()); + config.local_cache().borrow_mut().put( + "abc123digest".to_string(), + CachedPolicy::new(expected_policy.to_string(), Instant::now()), + ); let mut mock_filter = MockEnvoyHttpFilter::new(); mock_filter @@ -554,6 +592,49 @@ mod tests { ); } + #[test] + fn test_on_request_headers_expired_cache_entry_is_removed_and_increments_miss_counter() { + let hit_id = EnvoyCounterId(1); + let miss_id = EnvoyCounterId(2); + let config = EgressPolicyCacheFilterConfig::with_counters( + Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: true, + max_cache_items: 1000, + }, + hit_id, + miss_id, + ); + config.local_cache().borrow_mut().put( + "abc123digest".to_string(), + CachedPolicy::new( + r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#.to_string(), + Instant::now() - Duration::from_secs(10), + ), + ); + + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_attribute_string() + .withf(|id| { + *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest + }) + .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + mock_filter.expect_set_filter_state_bytes().never(); + mock_filter + .expect_increment_counter() + .withf(move |id, val| *id == miss_id && *val == 1) + .return_const(Result::<(), envoy_dynamic_module_type_metrics_result>::Ok(())) + .once(); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_request_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue + ); + assert!(config.local_cache().borrow_mut().get("abc123digest").is_none()); + } + #[test] fn test_on_response_headers_caches_on_200_with_policy_and_cert_digest() { let config = EgressPolicyCacheFilterConfig::new(Config::default()); @@ -562,6 +643,10 @@ mod tests { .expect_get_response_header_value() .withf(|key| key == ":status") .returning(|_| Some(EnvoyBuffer::new(b"200"))); + mock_filter + .expect_get_filter_state_bytes() + .withf(|key| key == ATE_POLICY_EGRESS_CACHED.as_bytes()) + .returning(|_| None); mock_filter .expect_get_filter_state_bytes() .withf(|key| key == ATE_POLICY_EGRESS.as_bytes()) @@ -583,14 +668,58 @@ mod tests { envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue ); + let cached = config + .local_cache() + .borrow_mut() + .get("abc123digest") + .cloned() + .expect("expected entry in cache"); assert_eq!( - config - .local_cache() - .borrow_mut() - .get("abc123digest") - .map(String::as_str), - Some(r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#) + cached.policy, + r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"# ); + assert!(cached.stored_at.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn test_on_response_headers_skips_cache_when_already_cached() { + let config = EgressPolicyCacheFilterConfig::new(Config::default()); + let initial_stored_at = Instant::now() - Duration::from_secs(3); + config.local_cache().borrow_mut().put( + "abc123digest".to_string(), + CachedPolicy::new( + r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#.to_string(), + initial_stored_at, + ), + ); + + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter + .expect_get_response_header_value() + .withf(|key| key == ":status") + .returning(|_| Some(EnvoyBuffer::new(b"200"))); + mock_filter + .expect_get_filter_state_bytes() + .withf(|key| key == ATE_POLICY_EGRESS_CACHED.as_bytes()) + .returning(|_| { + Some(EnvoyBuffer::new( + br#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#, + )) + }); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_response_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + ); + + let cached = config + .local_cache() + .borrow_mut() + .get("abc123digest") + .cloned() + .expect("expected entry in cache"); + assert_eq!(cached.stored_at, initial_stored_at); } #[test] @@ -633,18 +762,23 @@ mod tests { assert_eq!(filter1.local_cache().borrow().cap().get(), 2); + let now = Instant::now(); filter1 .local_cache() .borrow_mut() - .put("k1".to_string(), "v1".to_string()); + .put("k1".to_string(), CachedPolicy::new("v1".to_string(), now)); filter1 .local_cache() .borrow_mut() - .put("k2".to_string(), "v2".to_string()); + .put("k2".to_string(), CachedPolicy::new("v2".to_string(), now)); // Second filter created from the same config on the same thread sees the entries. assert_eq!( - filter2.local_cache().borrow_mut().get("k1").map(String::as_str), + filter2 + .local_cache() + .borrow_mut() + .get("k1") + .map(|e| e.policy.as_str()), Some("v1") ); @@ -652,14 +786,22 @@ mod tests { filter2 .local_cache() .borrow_mut() - .put("k3".to_string(), "v3".to_string()); + .put("k3".to_string(), CachedPolicy::new("v3".to_string(), now)); assert_eq!(filter1.local_cache().borrow_mut().get("k2"), None); assert_eq!( - filter1.local_cache().borrow_mut().get("k1").map(String::as_str), + filter1 + .local_cache() + .borrow_mut() + .get("k1") + .map(|e| e.policy.as_str()), Some("v1") ); assert_eq!( - filter1.local_cache().borrow_mut().get("k3").map(String::as_str), + filter1 + .local_cache() + .borrow_mut() + .get("k3") + .map(|e| e.policy.as_str()), Some("v3") ); diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index aa83424fa3..ee133f716c 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -203,6 +203,61 @@ func TestActorEgressPolicyCache(t *testing.T) { } } +// TestActorEgressPolicyCacheExpiration verifies that cached egress policy +// entries expire after the configured cache_ttl (5s default), triggering a +// cache miss and a new ext_proc lookup on the next request. +func TestActorEgressPolicyCacheExpiration(t *testing.T) { + ctx := context.Background() + router, actorRef := hostnamePolicyActor(t, ctx) + + beforeScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) + } + beforeHits, beforeMisses := e2e.EgressPolicyCacheCounts(beforeScrape) + beforeExtProc := e2e.EgressExtProcStreamCounts(beforeScrape) + + payload := []byte(`{"url":"https://example.com/","disableKeepAlive":true}`) + for i := range 2 { + status, body := postThroughEgressActorUntil(t, ctx, router, actorRef, "/", payload, reached) + if status != http.StatusOK { + t.Fatalf("pre-expiration request %d to example.com returned HTTP %d, want 200; body: %s", i+1, status, body) + } + } + + // Wait longer than the 5s default cache_ttl so the cached policy expires. + time.Sleep(6 * time.Second) + + for i := range 2 { + status, body := postThroughEgressActorUntil(t, ctx, router, actorRef, "/", payload, reached) + if status != http.StatusOK { + t.Fatalf("post-expiration request %d to example.com returned HTTP %d, want 200; body: %s", i+1, status, body) + } + } + + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) + } + afterHits, afterMisses := e2e.EgressPolicyCacheCounts(afterScrape) + afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) + + const ( + wantHits = 2 + wantMisses = 2 + wantExtProcCalls = 2 + ) + if got := afterHits - beforeHits; got != wantHits { + t.Errorf("egress policy cache hits delta = %d, want %d", got, wantHits) + } + if got := afterMisses - beforeMisses; got != wantMisses { + t.Errorf("egress policy cache misses delta = %d, want %d", got, wantMisses) + } + if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantExtProcCalls { + t.Errorf("egress_identity ext_proc streams_started delta = %d, want %d", got, wantExtProcCalls) + } +} + // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller // deciding which answer is final. func fetchThroughEgressActorUntil(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, url string, done func(status int, body []byte) bool) (int, []byte) { From 94c50b94d32e2d9265b1567c633fea3a0abf66f9 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 14:00:53 +0000 Subject: [PATCH 06/23] Implement cache enabling flag Signed-off-by: Yan Avlasov --- .../egress-policy-cache/src/lib.rs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 1512b93674..329c37634d 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -244,7 +244,6 @@ impl HttpFilter for EgressPolicyCacheFilter { envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let _ = self.config.cache_enabled; let cached_policy = if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { let mut cache = self.local_cache().borrow_mut(); if let Some(entry) = cache.get(&cert_digest) { @@ -278,7 +277,8 @@ impl HttpFilter for EgressPolicyCacheFilter { // Store policy in cache in case it was authorized (200 response) on a // cache miss. Actor identity comes from cert's subject. Use cert digest as // key, since it uniquely identifies the cert and hence actor. - if is_http_200(envoy_filter) + if self.config.cache_enabled + && is_http_200(envoy_filter) && !has_cached_egress_policy(envoy_filter) && let Some(policy) = read_egress_policy(envoy_filter) && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) @@ -722,6 +722,24 @@ mod tests { assert_eq!(cached.stored_at, initial_stored_at); } + #[test] + fn test_on_response_headers_skips_cache_when_disabled() { + let config = EgressPolicyCacheFilterConfig::new(Config { + cache_ttl: Duration::from_secs(5), + cache_enabled: false, + max_cache_items: 1000, + }); + let mut mock_filter = MockEnvoyHttpFilter::new(); + mock_filter.expect_get_response_header_value().never(); + + let mut filter = config.new_http_filter(&mut mock_filter); + assert_eq!( + filter.on_response_headers(&mut mock_filter, false), + envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue + ); + assert_eq!(config.local_cache().borrow().len(), 0); + } + #[test] fn test_on_response_headers_skips_cache_on_non_200() { let config = EgressPolicyCacheFilterConfig::new(Config::default()); From 22caefbf896340cb1bb43171ab252f10d942592d Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 14:12:15 +0000 Subject: [PATCH 07/23] Fixes Signed-off-by: Yan Avlasov --- .../egress-policy-cache/src/lib.rs | 70 +++++++------------ .../e2e/suites/egressmitm/egressmitm_test.go | 3 +- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 329c37634d..eaee20fc4d 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -175,16 +175,22 @@ impl EgressPolicyCacheFilterConfig { let max_items = self.config.max_cache_items; self.cache.get_or(|| new_lru_cache(max_items)) } -} -impl HttpFilterConfig for EgressPolicyCacheFilterConfig { - fn new_http_filter(&self, _envoy: &mut EHF) -> Box> { - Box::new(EgressPolicyCacheFilter { + /// Creates a concrete [`EgressPolicyCacheFilter`] bound to this configuration. + pub fn create_filter(&self) -> EgressPolicyCacheFilter { + EgressPolicyCacheFilter { config: self.config.clone(), cache: Arc::clone(&self.cache), cache_hit_counter: self.cache_hit_counter, cache_miss_counter: self.cache_miss_counter, - }) + has_cached_policy: false, + } + } +} + +impl HttpFilterConfig for EgressPolicyCacheFilterConfig { + fn new_http_filter(&self, _envoy: &mut EHF) -> Box> { + Box::new(self.create_filter()) } } @@ -194,6 +200,7 @@ pub struct EgressPolicyCacheFilter { pub cache: Arc, pub cache_hit_counter: EnvoyCounterId, pub cache_miss_counter: EnvoyCounterId, + pub has_cached_policy: bool, } impl EgressPolicyCacheFilter { @@ -212,12 +219,6 @@ fn is_http_200(envoy_filter: &EHF) -> bool { false } -fn has_cached_egress_policy(envoy_filter: &EHF) -> bool { - envoy_filter - .get_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes()) - .is_some() -} - fn read_egress_policy(envoy_filter: &EHF) -> Option { if let Some(buf) = envoy_filter.get_filter_state_bytes(ATE_POLICY_EGRESS.as_bytes()) && let Ok(s) = std::str::from_utf8(buf.as_slice()) @@ -261,6 +262,7 @@ impl HttpFilter for EgressPolicyCacheFilter { }; if let Some(policy) = cached_policy { + self.has_cached_policy = true; envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes(), policy.as_bytes()); let _ = envoy_filter.increment_counter(self.cache_hit_counter, 1); } else { @@ -278,8 +280,8 @@ impl HttpFilter for EgressPolicyCacheFilter { // cache miss. Actor identity comes from cert's subject. Use cert digest as // key, since it uniquely identifies the cert and hence actor. if self.config.cache_enabled + && !self.has_cached_policy && is_http_200(envoy_filter) - && !has_cached_egress_policy(envoy_filter) && let Some(policy) = read_egress_policy(envoy_filter) && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { @@ -585,11 +587,13 @@ mod tests { .return_const(Result::<(), envoy_dynamic_module_type_metrics_result>::Ok(())) .once(); - let mut filter = config.new_http_filter(&mut mock_filter); + let mut filter = config.create_filter(); + assert!(!filter.has_cached_policy); assert_eq!( filter.on_request_headers(&mut mock_filter, false), envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue ); + assert!(filter.has_cached_policy); } #[test] @@ -627,11 +631,12 @@ mod tests { .return_const(Result::<(), envoy_dynamic_module_type_metrics_result>::Ok(())) .once(); - let mut filter = config.new_http_filter(&mut mock_filter); + let mut filter = config.create_filter(); assert_eq!( filter.on_request_headers(&mut mock_filter, false), envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue ); + assert!(!filter.has_cached_policy); assert!(config.local_cache().borrow_mut().get("abc123digest").is_none()); } @@ -643,10 +648,6 @@ mod tests { .expect_get_response_header_value() .withf(|key| key == ":status") .returning(|_| Some(EnvoyBuffer::new(b"200"))); - mock_filter - .expect_get_filter_state_bytes() - .withf(|key| key == ATE_POLICY_EGRESS_CACHED.as_bytes()) - .returning(|_| None); mock_filter .expect_get_filter_state_bytes() .withf(|key| key == ATE_POLICY_EGRESS.as_bytes()) @@ -698,16 +699,10 @@ mod tests { .expect_get_response_header_value() .withf(|key| key == ":status") .returning(|_| Some(EnvoyBuffer::new(b"200"))); - mock_filter - .expect_get_filter_state_bytes() - .withf(|key| key == ATE_POLICY_EGRESS_CACHED.as_bytes()) - .returning(|_| { - Some(EnvoyBuffer::new( - br#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#, - )) - }); + mock_filter.expect_get_filter_state_bytes().never(); - let mut filter = config.new_http_filter(&mut mock_filter); + let mut filter = config.create_filter(); + filter.has_cached_policy = true; assert_eq!( filter.on_response_headers(&mut mock_filter, false), envoy_dynamic_module_type_on_http_filter_response_headers_status::Continue @@ -765,18 +760,8 @@ mod tests { max_cache_items: 2, })); - let filter1 = EgressPolicyCacheFilter { - config: filter_config.config.clone(), - cache: Arc::clone(&filter_config.cache), - cache_hit_counter: filter_config.cache_hit_counter, - cache_miss_counter: filter_config.cache_miss_counter, - }; - let filter2 = EgressPolicyCacheFilter { - config: filter_config.config.clone(), - cache: Arc::clone(&filter_config.cache), - cache_hit_counter: filter_config.cache_hit_counter, - cache_miss_counter: filter_config.cache_miss_counter, - }; + let filter1 = filter_config.create_filter(); + let filter2 = filter_config.create_filter(); assert_eq!(filter1.local_cache().borrow().cap().get(), 2); @@ -826,12 +811,7 @@ mod tests { // A filter on another worker thread gets its own empty thread-local cache. let filter_config_clone = Arc::clone(&filter_config); let other_thread_len = std::thread::spawn(move || { - let other_filter = EgressPolicyCacheFilter { - config: filter_config_clone.config.clone(), - cache: Arc::clone(&filter_config_clone.cache), - cache_hit_counter: filter_config_clone.cache_hit_counter, - cache_miss_counter: filter_config_clone.cache_miss_counter, - }; + let other_filter = filter_config_clone.create_filter(); assert_eq!(other_filter.local_cache().borrow().cap().get(), 2); other_filter.local_cache().borrow().len() }) diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 0dafacf736..b15b31daa3 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -187,8 +187,9 @@ func TestActorEgressMITMTrust(t *testing.T) { // Each of the 6 fetches opens a CONNECT tunnel (egress_identity), while // only the positive MITM fetch completes the inner TLS handshake and sends // an HTTP request through the MITM chain (egress_policy_mitm). + // Since caching is enabled only the first CONNECT request will result in ext_proc call. const ( - wantIdentityCalls = 6 + wantIdentityCalls = 1 wantMITMPolicyCalls = 1 ) if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantIdentityCalls { From e9b26884f8475299a0c565fdf1dfd749d6cf03d2 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 14:28:29 +0000 Subject: [PATCH 08/23] Docs Signed-off-by: Yan Avlasov --- .../egress-policy-cache/README.md | 189 ++++++++++++++++++ .../egress-policy-cache/src/lib.rs | 4 + 2 files changed, 193 insertions(+) create mode 100644 cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md new file mode 100644 index 0000000000..0b1e19cffd --- /dev/null +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -0,0 +1,189 @@ +# Envoy Substrate Egress Policy Cache - Rust Dynamic Module + +An Envoy dynamic module, written in Rust, that runs as an HTTP filter on the +egress gateway's outer `CONNECT` listener and caches egress policy SNI rules +per actor certificate in a thread-local LRU cache so repeat `CONNECT` requests +from the same actor can bypass the `ext_proc` sidecar. + +## How it works + +Each `EgressPolicyCacheFilterConfig` owns a `ThreadLocal` store of +`LruCache` instances (`max_cache_items` capacity per +Envoy worker thread). Each `CachedPolicy` entry stores the serialized egress +policy JSON string alongside the `Instant` (`stored_at`) when it was cached, +keyed by the downstream peer certificate's SHA-256 digest +(`connection.sha256_peer_certificate_digest`). + +### Request path (`on_request_headers`) + +When an actor opens a `CONNECT` tunnel on the outer listener: + +1. If `cache_enabled` is `false`, the filter returns `Continue` immediately. +2. The filter reads `connection.sha256_peer_certificate_digest` and looks up + the digest in the current worker thread's LRU cache: + - **Fresh hit (`stored_at.elapsed() <= cache_ttl`)**: + - Writes the cached policy JSON to filter state under key + `dev.ate.policy.egress.cached`. + - Sets `has_cached_policy = true` on the per-stream + `EgressPolicyCacheFilter`. + - Increments the `ate_egress.connect_cache_hit` counter. + - **Expired entry (`stored_at.elapsed() > cache_ttl`)**: + - Removes the expired entry from the LRU cache without setting filter + state. + - Increments the `ate_egress.connect_cache_miss` counter. + - **Cache miss (or missing certificate digest)**: + - Increments the `ate_egress.connect_cache_miss` counter. +3. Subsequent filters in the outer HTTP filter chain act on the result: + - `envoy.filters.http.composite` checks for the presence of + `dev.ate.policy.egress.cached` in filter state and invokes + `envoy.filters.http.ext_proc` only when it is absent. + - `envoy.filters.http.set_filter_state` copies either + `%DYNAMIC_METADATA(dev.ate.policy.egress)%` (on a cache miss, populated by + `ext_proc`) or `%FILTER_STATE(dev.ate.policy.egress.cached:PLAIN)%` (on a + cache hit) into the `dev.ate.policy.egress` filter state with + `shared_with_upstream: ONCE`, making it available to the inner listener's + `egress-policy` listener filter. + +### Response path (`on_response_headers`) + +When the `CONNECT` response headers arrive: + +1. If `cache_enabled` is `false`, the filter returns `Continue` immediately. +2. If `:status` is `200` and `!self.has_cached_policy`: + - Reads the egress policy JSON from filter state key + `dev.ate.policy.egress` and the peer certificate digest from + `connection.sha256_peer_certificate_digest`. + - Inserts `CachedPolicy { policy, stored_at: Instant::now() }` into the + current worker thread's LRU cache. +3. If `self.has_cached_policy` is `true` (the policy for this stream came from + the cache), caching is skipped so the entry's original `stored_at` timestamp + is preserved until `cache_ttl` expires. + +## Filter configuration + +The filter accepts an optional JSON configuration object in `filter_config` +(an empty config defaults to `{}`): + +| Field | Type | Default | Description | +|---|---|---|---| +| `cache_ttl` | `u64` (seconds) | `5` | Time-to-live in seconds for cached policy entries before they expire | +| `cache_enabled` | `bool` | `true` | Enables or disables reading from and writing to the policy cache | +| `max_cache_items` (or `max-cache-items`) | `usize` | `1000` | Maximum number of entries in each worker thread's LRU cache | + +Example JSON configuration: + +```json +{ + "cache_ttl": 5, + "cache_enabled": true, + "max_cache_items": 1000 +} +``` + +## Metrics + +The filter defines two Envoy counters on config initialization: + +| Counter name | Envoy stat name | Description | +|---|---|---| +| `ate_egress.connect_cache_hit` | `dynamic_modules.custom.egress_policy_cache.ate_egress.connect_cache_hit` | Incremented when a non-expired policy is found in the cache on `CONNECT` | +| `ate_egress.connect_cache_miss` | `dynamic_modules.custom.egress_policy_cache.ate_egress.connect_cache_miss` | Incremented when no entry or an expired entry is found in the cache on `CONNECT` | + +## Building + +Prerequisites: Rust toolchain (Cargo, rustc 1.75+), plus `clang` and +`libclang-dev` for the SDK's bindgen step. + +```bash +cargo build --release +``` + +The compiled shared object will be located at +`target/release/libenvoy_substrate_egress_policy_cache.so`. +`cmd/dataplane/envoy/Dockerfile` builds it and packages it into the Envoy +dataplane image. + +## Testing + +```bash +cargo test +``` + +Or from the repository root: + +```bash +hack/test-dynamic-modules.sh +``` + +> **Note:** Because the LRU cache is thread-local per Envoy worker thread, set +> `export E2E_ENVOY_CONCURRENCY=1` before deploying `ate-system` when running +> the networking E2E suite (`./internal/e2e/suites/networking`) to test caching +> behavior (`TestActorEgressPolicyCache` and +> `TestActorEgressPolicyCacheExpiration`). This configures Envoy with a single +> worker thread (`--concurrency 1`) so consecutive `CONNECT` requests are +> routed to the same thread-local cache instance. + +## Envoy configuration + +In the outer `CONNECT` listener's `http_filters`, place the dynamic module +filter before the conditional `ext_proc` composite filter and the post-`ext_proc` +`set_filter_state` filter: + +```yaml +- name: envoy.filters.http.dynamic_modules + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_modules.v3.DynamicModuleFilter + dynamic_module_config: + name: envoy_substrate_egress_policy_cache + filter_name: envoy_substrate_egress_policy_cache +- name: envoy.filters.http.composite + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite + matcher: + matcher_list: + matchers: + - predicate: + not_matcher: + single_predicate: + input: + name: filter_state + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.FilterStateInput + key: dev.ate.policy.egress.cached + value_match: + safe_regex: + google_re2: {} + regex: ".*" + on_match: + action: + name: composite-action + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.ExecuteFilterAction + typed_config: + name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + # ... +- name: envoy.filters.http.set_filter_state + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config + on_request_headers: + - object_key: dev.ate.policy.egress + factory_key: envoy.string + skip_if_empty: true + shared_with_upstream: ONCE + format_string: + text_format_source: + inline_string: "%DYNAMIC_METADATA(dev.ate.policy.egress)%" + omit_empty_values: true + - object_key: dev.ate.policy.egress + factory_key: envoy.string + skip_if_empty: true + shared_with_upstream: ONCE + format_string: + text_format_source: + inline_string: "%FILTER_STATE(dev.ate.policy.egress.cached:PLAIN)%" + omit_empty_values: true +``` + +See `manifests/ate-install/atenet-egress.yaml` for the complete configuration. diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index eaee20fc4d..258c2d2487 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -36,6 +36,10 @@ pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; /// Key holding the cached egress policy SNI rules JSON before it is copied to /// upstream-shared filter state. +// TODO(yanavlasov): this is a temporary workaround of the Rust dynamic module API +// limitation that does not allow storing filter state shared with upstream. +// The dev.ate.policy.egress.cached filter state is later copied into +// dev.ate.policy.egress filter state by the set_filter_state HTTP filter. pub const ATE_POLICY_EGRESS_CACHED: &str = "dev.ate.policy.egress.cached"; /// Counter name for egress policy cache hits on CONNECT. From d8e057c1bcd7ba03ff5297f87e6eebd51202ba50 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 16:18:31 +0000 Subject: [PATCH 09/23] Skip tests if concurrency is not set Signed-off-by: Yan Avlasov --- internal/e2e/atenet_dataplane.go | 7 +++++++ internal/e2e/suites/egressmitm/egressmitm_test.go | 5 +++++ internal/e2e/suites/networking/egresspolicy_test.go | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index 7f2663048d..8b1c487587 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -39,6 +39,7 @@ type AtenetDataplane interface { PlatformMetricPrefixes([]string) []string RouteDurationSeen(context.Context, string) (bool, error) SupportsIngressProtocolDowngrade() bool + IsDataplaneConcurrencyDisabled() bool } // CurrentAtenetDataplane returns the implementation selected for this test @@ -83,6 +84,10 @@ func (envoyAtenetDataplane) RouteDurationSeen(_ context.Context, collectorScrape func (envoyAtenetDataplane) SupportsIngressProtocolDowngrade() bool { return true } +func (envoyAtenetDataplane) IsDataplaneConcurrencyDisabled() bool { + return os.Getenv("E2E_ENVOY_CONCURRENCY") == "1" +} + type agentGatewayAtenetDataplane struct{} func (agentGatewayAtenetDataplane) NewParkingObserver(context.Context) (ParkingObserver, error) { @@ -122,6 +127,8 @@ func (agentGatewayAtenetDataplane) RouteDurationSeen(ctx context.Context, _ stri func (agentGatewayAtenetDataplane) SupportsIngressProtocolDowngrade() bool { return false } +func (agentGatewayAtenetDataplane) IsDataplaneConcurrencyDisabled() bool { return false } + type agentGatewayParkingObserver struct{} func (agentGatewayParkingObserver) WaitForCount(ctx context.Context, cond func(int) bool) (int, error) { diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index b15b31daa3..3ec292464c 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -178,6 +178,11 @@ func TestActorEgressMITMTrust(t *testing.T) { t.Errorf("fetch of a host outside the policy got status %s with error %q, want no HTTP exchange at all", denied.Status, denied.Error) } + if !e2e.CurrentAtenetDataplane().IsDataplaneConcurrencyDisabled() { + t.Log("skipping counter verification: dataplane concurrency is not disabled") + return + } + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) if err != nil { t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index ee133f716c..46b8dc0cba 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -180,6 +180,11 @@ func TestActorEgressPolicyCache(t *testing.T) { } } + if !e2e.CurrentAtenetDataplane().IsDataplaneConcurrencyDisabled() { + t.Log("skipping counter verification: dataplane concurrency is not disabled") + return + } + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) if err != nil { t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) @@ -235,6 +240,11 @@ func TestActorEgressPolicyCacheExpiration(t *testing.T) { } } + if !e2e.CurrentAtenetDataplane().IsDataplaneConcurrencyDisabled() { + t.Log("skipping counter verification: dataplane concurrency is not disabled") + return + } + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) if err != nil { t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) From 12786f84d205453cc9bf5b8c5e97c606b8d519de Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 16:28:47 +0000 Subject: [PATCH 10/23] cleanup Signed-off-by: Yan Avlasov --- cmd/atenet/internal/router/extproc/attributes.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index da414e6fca..e4ac2944ca 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -127,13 +127,6 @@ const UpstreamDynamicPortFilterStateKey = "envoy.upstream.dynamic_port" // EgressPolicyMetadataFormat renders EgressPolicyMetadataNamespace as JSON. const EgressPolicyMetadataFormat = "%DYNAMIC_METADATA(" + EgressPolicyMetadataNamespace + ")%" -// EgressPolicyCachedFilterStateKey holds the cached SNI rules written by the -// egress-policy-cache module before set_filter_state shares them upstream. -const EgressPolicyCachedFilterStateKey = EgressPolicyMetadataNamespace + ".cached" - -// EgressPolicyCachedFormat reads EgressPolicyCachedFilterStateKey as a plain string. -const EgressPolicyCachedFormat = "%FILTER_STATE(" + EgressPolicyCachedFilterStateKey + ":PLAIN)%" - // OriginalDstFilterStateKey is Envoy's filter-state key for the address an // ORIGINAL_DST cluster dials. The gateway never writes it and dials only by // name; the request legs read it as an attribute. From 8cb7351a0038a160f91559e432501365e2417d06 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 18:36:07 +0000 Subject: [PATCH 11/23] Add dest port to cache key Signed-off-by: Yan Avlasov --- .../egress-policy-cache/README.md | 21 ++--- .../egress-policy-cache/src/lib.rs | 80 ++++++++++++++++--- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md index 0b1e19cffd..da2973ac5e 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -11,16 +11,18 @@ Each `EgressPolicyCacheFilterConfig` owns a `ThreadLocal` store of `LruCache` instances (`max_cache_items` capacity per Envoy worker thread). Each `CachedPolicy` entry stores the serialized egress policy JSON string alongside the `Instant` (`stored_at`) when it was cached, -keyed by the downstream peer certificate's SHA-256 digest -(`connection.sha256_peer_certificate_digest`). +keyed by `;`, where `` is +the downstream peer certificate's SHA-256 digest +(`connection.sha256_peer_certificate_digest`) and `` is the +destination port extracted from the `:authority` request header. ### Request path (`on_request_headers`) When an actor opens a `CONNECT` tunnel on the outer listener: -1. If `cache_enabled` is `false`, the filter returns `Continue` immediately. -2. The filter reads `connection.sha256_peer_certificate_digest` and looks up - the digest in the current worker thread's LRU cache: +1. The filter builds the cache key `;` from + `connection.sha256_peer_certificate_digest` and the `:authority` request + header, and looks it up in the current worker thread's LRU cache: - **Fresh hit (`stored_at.elapsed() <= cache_ttl`)**: - Writes the cached policy JSON to filter state under key `dev.ate.policy.egress.cached`. @@ -31,9 +33,9 @@ When an actor opens a `CONNECT` tunnel on the outer listener: - Removes the expired entry from the LRU cache without setting filter state. - Increments the `ate_egress.connect_cache_miss` counter. - - **Cache miss (or missing certificate digest)**: + - **Cache miss (or missing certificate digest / destination port)**: - Increments the `ate_egress.connect_cache_miss` counter. -3. Subsequent filters in the outer HTTP filter chain act on the result: +2. Subsequent filters in the outer HTTP filter chain act on the result: - `envoy.filters.http.composite` checks for the presence of `dev.ate.policy.egress.cached` in filter state and invokes `envoy.filters.http.ext_proc` only when it is absent. @@ -51,8 +53,9 @@ When the `CONNECT` response headers arrive: 1. If `cache_enabled` is `false`, the filter returns `Continue` immediately. 2. If `:status` is `200` and `!self.has_cached_policy`: - Reads the egress policy JSON from filter state key - `dev.ate.policy.egress` and the peer certificate digest from - `connection.sha256_peer_certificate_digest`. + `dev.ate.policy.egress` and builds the cache key + `;` from + `connection.sha256_peer_certificate_digest` and `:authority`. - Inserts `CachedPolicy { policy, stored_at: Instant::now() }` into the current worker thread's LRU cache. 3. If `self.has_cached_policy` is `true` (the policy for this stream came from diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 258c2d2487..11a9cc510e 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -243,17 +243,41 @@ fn read_peer_cert_digest(envoy_filter: &EHF) -> Option Option<&str> { + let (host, port) = authority.rsplit_once(':')?; + if host.is_empty() || (host.contains(':') && !host.ends_with(']')) { + return None; + } + let port_num: u16 = port.parse().ok()?; + if port_num == 0 { + return None; + } + Some(port) +} + +fn read_destination_port(envoy_filter: &EHF) -> Option { + let buf = envoy_filter.get_request_header_value(":authority")?; + let authority = std::str::from_utf8(buf.as_slice()).ok()?; + extract_destination_port(authority).map(str::to_owned) +} + +fn build_cache_key(envoy_filter: &EHF) -> Option { + let cert_digest = read_peer_cert_digest(envoy_filter)?; + let port = read_destination_port(envoy_filter)?; + Some(format!("{cert_digest};{port}")) +} + impl HttpFilter for EgressPolicyCacheFilter { fn on_request_headers( &mut self, envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let cached_policy = if let Some(cert_digest) = read_peer_cert_digest(envoy_filter) { + let cached_policy = if let Some(cache_key) = build_cache_key(envoy_filter) { let mut cache = self.local_cache().borrow_mut(); - if let Some(entry) = cache.get(&cert_digest) { + if let Some(entry) = cache.get(&cache_key) { if entry.stored_at.elapsed() > self.config.cache_ttl { - cache.pop(&cert_digest); + cache.pop(&cache_key); None } else { Some(entry.policy.clone()) @@ -281,16 +305,17 @@ impl HttpFilter for EgressPolicyCacheFilter { _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_response_headers_status { // Store policy in cache in case it was authorized (200 response) on a - // cache miss. Actor identity comes from cert's subject. Use cert digest as - // key, since it uniquely identifies the cert and hence actor. + // cache miss. Actor identity comes from cert's subject. Use cert digest + // concatenated with the destination port as key, since SNI rules depend on + // both the actor identity and the dialed port. if self.config.cache_enabled && !self.has_cached_policy && is_http_200(envoy_filter) && let Some(policy) = read_egress_policy(envoy_filter) - && let Some(cert_digest) = read_peer_cert_digest(envoy_filter) + && let Some(cache_key) = build_cache_key(envoy_filter) { self.local_cache().borrow_mut().put( - cert_digest, + cache_key, CachedPolicy::new(policy, Instant::now()), ); } @@ -525,6 +550,19 @@ mod tests { ); } + #[test] + fn test_extract_destination_port() { + assert_eq!(extract_destination_port("10.0.0.1:443"), Some("443")); + assert_eq!(extract_destination_port("[2001:db8::1]:8443"), Some("8443")); + assert_eq!(extract_destination_port("example.com:80"), Some("80")); + assert_eq!(extract_destination_port("example.com"), None); + assert_eq!(extract_destination_port(":443"), None); + assert_eq!(extract_destination_port("10.0.0.1:0"), None); + assert_eq!(extract_destination_port("10.0.0.1:70000"), None); + assert_eq!(extract_destination_port("2001:db8::1"), None); + assert_eq!(extract_destination_port("[2001:db8::1]"), None); + } + #[test] fn test_headers_continue() { let hit_id = EnvoyCounterId(1); @@ -538,6 +576,10 @@ mod tests { *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest }) .returning(|_| Some(EnvoyBuffer::new(b"unknown_digest"))); + mock_filter + .expect_get_request_header_value() + .withf(|key| key == ":authority") + .returning(|_| Some(EnvoyBuffer::new(b"10.0.0.1:443"))); mock_filter .expect_increment_counter() .withf(move |id, val| *id == miss_id && *val == 1) @@ -567,7 +609,7 @@ mod tests { EgressPolicyCacheFilterConfig::with_counters(Config::default(), hit_id, miss_id); let expected_policy = r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#; config.local_cache().borrow_mut().put( - "abc123digest".to_string(), + "abc123digest;443".to_string(), CachedPolicy::new(expected_policy.to_string(), Instant::now()), ); @@ -578,6 +620,10 @@ mod tests { *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest }) .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + mock_filter + .expect_get_request_header_value() + .withf(|key| key == ":authority") + .returning(|_| Some(EnvoyBuffer::new(b"10.0.0.1:443"))); mock_filter .expect_set_filter_state_bytes() .withf(move |key, val| { @@ -614,7 +660,7 @@ mod tests { miss_id, ); config.local_cache().borrow_mut().put( - "abc123digest".to_string(), + "abc123digest;443".to_string(), CachedPolicy::new( r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#.to_string(), Instant::now() - Duration::from_secs(10), @@ -628,6 +674,10 @@ mod tests { *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest }) .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + mock_filter + .expect_get_request_header_value() + .withf(|key| key == ":authority") + .returning(|_| Some(EnvoyBuffer::new(b"10.0.0.1:443"))); mock_filter.expect_set_filter_state_bytes().never(); mock_filter .expect_increment_counter() @@ -641,7 +691,7 @@ mod tests { envoy_dynamic_module_type_on_http_filter_request_headers_status::Continue ); assert!(!filter.has_cached_policy); - assert!(config.local_cache().borrow_mut().get("abc123digest").is_none()); + assert!(config.local_cache().borrow_mut().get("abc123digest;443").is_none()); } #[test] @@ -666,6 +716,10 @@ mod tests { *id == envoy_dynamic_module_type_attribute_id::ConnectionSha256PeerCertificateDigest }) .returning(|_| Some(EnvoyBuffer::new(b"abc123digest"))); + mock_filter + .expect_get_request_header_value() + .withf(|key| key == ":authority") + .returning(|_| Some(EnvoyBuffer::new(b"10.0.0.1:443"))); let mut filter = config.new_http_filter(&mut mock_filter); assert_eq!( @@ -676,7 +730,7 @@ mod tests { let cached = config .local_cache() .borrow_mut() - .get("abc123digest") + .get("abc123digest;443") .cloned() .expect("expected entry in cache"); assert_eq!( @@ -691,7 +745,7 @@ mod tests { let config = EgressPolicyCacheFilterConfig::new(Config::default()); let initial_stored_at = Instant::now() - Duration::from_secs(3); config.local_cache().borrow_mut().put( - "abc123digest".to_string(), + "abc123digest;443".to_string(), CachedPolicy::new( r#"{"rules":[{"pattern":"*.example.com","mode":"mitm"}]}"#.to_string(), initial_stored_at, @@ -715,7 +769,7 @@ mod tests { let cached = config .local_cache() .borrow_mut() - .get("abc123digest") + .get("abc123digest;443") .cloned() .expect("expected entry in cache"); assert_eq!(cached.stored_at, initial_stored_at); From 7fed564fb0245816e7ff2002a9e1eba343c1c9c6 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Mon, 5 Oct 2026 18:36:57 +0000 Subject: [PATCH 12/23] fix tests Signed-off-by: Yan Avlasov --- cmd/atenet/internal/router/extproc/attributes.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index e4ac2944ca..da414e6fca 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -127,6 +127,13 @@ const UpstreamDynamicPortFilterStateKey = "envoy.upstream.dynamic_port" // EgressPolicyMetadataFormat renders EgressPolicyMetadataNamespace as JSON. const EgressPolicyMetadataFormat = "%DYNAMIC_METADATA(" + EgressPolicyMetadataNamespace + ")%" +// EgressPolicyCachedFilterStateKey holds the cached SNI rules written by the +// egress-policy-cache module before set_filter_state shares them upstream. +const EgressPolicyCachedFilterStateKey = EgressPolicyMetadataNamespace + ".cached" + +// EgressPolicyCachedFormat reads EgressPolicyCachedFilterStateKey as a plain string. +const EgressPolicyCachedFormat = "%FILTER_STATE(" + EgressPolicyCachedFilterStateKey + ":PLAIN)%" + // OriginalDstFilterStateKey is Envoy's filter-state key for the address an // ORIGINAL_DST cluster dials. The gateway never writes it and dials only by // name; the request legs read it as an attribute. From 251e3e7ff39bc1cab9d8d04310477d77ce3c4ebe Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Tue, 6 Oct 2026 01:30:18 +0000 Subject: [PATCH 13/23] Add per port policy cache test Signed-off-by: Yan Avlasov --- .../suites/networking/egresspolicy_test.go | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index beadcae073..30d20dbacb 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -99,7 +99,10 @@ func TestActorEgressRequiresPolicy(t *testing.T) { // example.com and nothing else, and waits until it is routable. func hostnamePolicyActor(t *testing.T, ctx context.Context) (*e2e.RouterClient, resources.ActorRef) { t.Helper() - _, actorName, _ := createAndResumeActorWithEgress(t, ctx, "egress-sni", e2e.EgressFixture(), e2e.EgressAllowHTTPS("example.com"), e2e.EgressAllowPassthrough("example.edu")) + _, actorName, _ := createAndResumeActorWithEgress(t, ctx, "egress-sni", e2e.EgressFixture(), + e2e.EgressAllowHTTPS("example.com"), + e2e.EgressAllowPassthrough("example.edu"), + e2e.EgressAllowHTTP("example.net")) router := mustRouterClient(t, ctx) t.Cleanup(func() { router.Close() }) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} @@ -265,6 +268,60 @@ func TestActorEgressPolicyCacheExpiration(t *testing.T) { } } +// TestActorEgressPolicyCachePerPort sends 5 requests to http://example.net and +// then 5 requests to https://example.com, verifying that per-port policy +// caching records 2 cache misses, 8 cache hits, and 2 outer CONNECT ext_proc +// callouts. +func TestActorEgressPolicyCachePerPort(t *testing.T) { + ctx := context.Background() + router, actorRef := hostnamePolicyActor(t, ctx) + + beforeScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) + } + beforeHits, beforeMisses := e2e.EgressPolicyCacheCounts(beforeScrape) + beforeExtProc := e2e.EgressExtProcStreamCounts(beforeScrape) + + const numRequests = 5 + for _, targetURL := range []string{"http://example.net/", "https://example.com/"} { + payload := []byte(fmt.Sprintf(`{"url":%q,"disableKeepAlive":true}`, targetURL)) + for i := range numRequests { + status, body := postThroughEgressActorUntil(t, ctx, router, actorRef, "/", payload, reached) + if status != http.StatusOK { + t.Fatalf("request %d to %s returned HTTP %d, want 200; body: %s", i+1, targetURL, status, body) + } + } + } + + if !e2e.CurrentAtenetDataplane().IsDataplaneConcurrencyDisabled() { + t.Log("skipping counter verification: dataplane concurrency is not disabled") + return + } + + afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) + if err != nil { + t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) + } + afterHits, afterMisses := e2e.EgressPolicyCacheCounts(afterScrape) + afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) + + const ( + wantHits = 8 + wantMisses = 2 + wantExtProcCalls = 2 + ) + if got := afterHits - beforeHits; got != wantHits { + t.Errorf("egress policy cache hits delta = %d, want %d", got, wantHits) + } + if got := afterMisses - beforeMisses; got != wantMisses { + t.Errorf("egress policy cache misses delta = %d, want %d", got, wantMisses) + } + if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantExtProcCalls { + t.Errorf("egress_identity ext_proc streams_started delta = %d, want %d", got, wantExtProcCalls) + } +} + // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller // deciding which answer is final. func fetchThroughEgressActorUntil(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, url string, done func(status int, body []byte) bool) (int, []byte) { From 9888426c96c077f927254e2a4e4e6669c5a2b8ac Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Tue, 6 Oct 2026 19:29:40 +0000 Subject: [PATCH 14/23] Add port setting metadata Signed-off-by: Yan Avlasov --- .../egress-policy-cache/README.md | 12 +++++- .../egress-policy-cache/src/lib.rs | 38 +++++++++++++++---- demos/egress/main.go | 9 ++++- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md index da2973ac5e..4faceef20c 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -2,8 +2,16 @@ An Envoy dynamic module, written in Rust, that runs as an HTTP filter on the egress gateway's outer `CONNECT` listener and caches egress policy SNI rules -per actor certificate in a thread-local LRU cache so repeat `CONNECT` requests -from the same actor can bypass the `ext_proc` sidecar. +per actor certificate and destination port in a thread-local LRU cache so +repeat `CONNECT` requests from the same actor to the same destination can +use locally cached policy and avoid external callout to get the policy. + +It is safe because external callout evaluates if there are policies that allow +given actor to egress to specific destination port and then provides SNI and host +matching rules to the dataplane. Since local cache is populated only when CONNECT +was allowed and keyed by the actor ID and destination port it safe to use cached +SNI and host matching rules. The freshness configuration `cache_ttl` determines +responsiveness of the dataplane to policy changes. ## How it works diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 11a9cc510e..4d08d425b7 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -36,12 +36,20 @@ pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; /// Key holding the cached egress policy SNI rules JSON before it is copied to /// upstream-shared filter state. -// TODO(yanavlasov): this is a temporary workaround of the Rust dynamic module API +// TODO(yanavlasov): this is a temporary workaround for the Rust dynamic module API // limitation that does not allow storing filter state shared with upstream. // The dev.ate.policy.egress.cached filter state is later copied into // dev.ate.policy.egress filter state by the set_filter_state HTTP filter. +// Once this limitation is addressed this filter can set "dev.ate.policy.egress" filter +// state directly and share it with upstream. pub const ATE_POLICY_EGRESS_CACHED: &str = "dev.ate.policy.egress.cached"; +/// Dynamic metadata namespace for egress attributes set on CONNECT. +pub const ATE_EGRESS_METADATA_NAMESPACE: &str = "dev.ate.egress"; + +/// Dynamic metadata key holding the destination port dialed by the actor. +pub const ATE_EGRESS_DIALED_PORT_KEY: &str = "dialed_port"; + /// Counter name for egress policy cache hits on CONNECT. pub const CONNECT_CACHE_HIT_COUNTER: &str = "ate_egress.connect_cache_hit"; @@ -261,10 +269,11 @@ fn read_destination_port(envoy_filter: &EHF) -> Option(envoy_filter: &EHF) -> Option { +fn build_cache_key(envoy_filter: &EHF) -> Option<(String, String)> { let cert_digest = read_peer_cert_digest(envoy_filter)?; let port = read_destination_port(envoy_filter)?; - Some(format!("{cert_digest};{port}")) + let key = format!("{cert_digest};{port}"); + Some((key, port)) } impl HttpFilter for EgressPolicyCacheFilter { @@ -273,14 +282,14 @@ impl HttpFilter for EgressPolicyCacheFilter { envoy_filter: &mut EHF, _end_of_stream: bool, ) -> envoy_dynamic_module_type_on_http_filter_request_headers_status { - let cached_policy = if let Some(cache_key) = build_cache_key(envoy_filter) { + let cached_policy = if let Some((cache_key, port)) = build_cache_key(envoy_filter) { let mut cache = self.local_cache().borrow_mut(); if let Some(entry) = cache.get(&cache_key) { if entry.stored_at.elapsed() > self.config.cache_ttl { cache.pop(&cache_key); None } else { - Some(entry.policy.clone()) + Some((entry.policy.clone(), port)) } } else { None @@ -289,9 +298,14 @@ impl HttpFilter for EgressPolicyCacheFilter { None }; - if let Some(policy) = cached_policy { + if let Some((policy, port)) = cached_policy { self.has_cached_policy = true; envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes(), policy.as_bytes()); + envoy_filter.set_dynamic_metadata_string( + ATE_EGRESS_METADATA_NAMESPACE, + ATE_EGRESS_DIALED_PORT_KEY, + &port, + ); let _ = envoy_filter.increment_counter(self.cache_hit_counter, 1); } else { let _ = envoy_filter.increment_counter(self.cache_miss_counter, 1); @@ -312,7 +326,7 @@ impl HttpFilter for EgressPolicyCacheFilter { && !self.has_cached_policy && is_http_200(envoy_filter) && let Some(policy) = read_egress_policy(envoy_filter) - && let Some(cache_key) = build_cache_key(envoy_filter) + && let Some((cache_key, _)) = build_cache_key(envoy_filter) { self.local_cache().borrow_mut().put( cache_key, @@ -631,6 +645,15 @@ mod tests { }) .return_const(true) .once(); + mock_filter + .expect_set_dynamic_metadata_string() + .withf(|namespace, key, val| { + namespace == ATE_EGRESS_METADATA_NAMESPACE + && key == ATE_EGRESS_DIALED_PORT_KEY + && val == "443" + }) + .return_const(()) + .once(); mock_filter .expect_increment_counter() .withf(move |id, val| *id == hit_id && *val == 1) @@ -679,6 +702,7 @@ mod tests { .withf(|key| key == ":authority") .returning(|_| Some(EnvoyBuffer::new(b"10.0.0.1:443"))); mock_filter.expect_set_filter_state_bytes().never(); + mock_filter.expect_set_dynamic_metadata_string().never(); mock_filter .expect_increment_counter() .withf(move |id, val| *id == miss_id && *val == 1) diff --git a/demos/egress/main.go b/demos/egress/main.go index 936b21ff84..dc76853f62 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -47,8 +47,13 @@ const ( ) type fetchRequest struct { - URL string `json:"url"` - DisableKeepAlive bool `json:"disableKeepAlive,omitempty"` + URL string `json:"url"` + // This flag allows tests to send each request over a new connection, + // resulting in a new CONNECT tunnel from the atunnel, rather than re-using + // existing connections in the http dialer connection pool and corresponding CONNECT + // tunnels in the atunnel. + // This is useful when testing egress policy caching behavior in the egress gateway. + DisableKeepAlive bool `json:"disableKeepAlive,omitempty"` } type fetchResponse struct { From 426b1e2d95d1bdbfa3a25f67161239678e47108a Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 00:41:24 +0000 Subject: [PATCH 15/23] Add dest port validation Signed-off-by: Yan Avlasov --- demos/egress/main.go | 6 ++++ demos/egress/main_test.go | 28 +++++++++++++++++++ .../suites/networking/egresspolicy_test.go | 28 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/demos/egress/main.go b/demos/egress/main.go index dc76853f62..123127ed45 100644 --- a/demos/egress/main.go +++ b/demos/egress/main.go @@ -48,6 +48,9 @@ const ( type fetchRequest struct { URL string `json:"url"` + // Host, when set, overrides the outbound HTTP request's Host header + // without changing the dial destination or TLS SNI derived from URL. + Host string `json:"host,omitempty"` // This flag allows tests to send each request over a new connection, // resulting in a new CONNECT tunnel from the atunnel, rather than re-using // existing connections in the http dialer connection pool and corresponding CONNECT @@ -139,6 +142,9 @@ func newHandler(client *http.Client) http.Handler { writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid URL: %v", err)}) return } + if input.Host != "" { + outbound.Host = input.Host + } if input.DisableKeepAlive { outbound.Close = true client.CloseIdleConnections() diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go index 6e60072dfd..bdb4ffe8b1 100644 --- a/demos/egress/main_test.go +++ b/demos/egress/main_test.go @@ -94,6 +94,34 @@ func TestFetchDisableKeepAlive(t *testing.T) { } } +func TestFetchHostOverride(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "allowed.example" { + t.Errorf("upstream URL.Host = %q, want %q", r.URL.Host, "allowed.example") + } + if r.Host != "allowed.example:1" { + t.Errorf("upstream Host = %q, want %q", r.Host, "allowed.example:1") + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("ok")), + Header: make(http.Header), + }, nil + })} + + payload, err := json.Marshal(fetchRequest{URL: "https://allowed.example/", Host: "allowed.example:1"}) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(payload))) + newHandler(client).ServeHTTP(recorder, request) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK) + } +} + func TestFetchHTTPAndHTTPS(t *testing.T) { httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "from http") diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 30d20dbacb..6dc7c7e528 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -322,6 +322,34 @@ func TestActorEgressPolicyCachePerPort(t *testing.T) { } } +// TestActorEgressDialedPortOverridesHostHeader sends requests to +// https://example.com/ (default TCP port 443) with "example.com:1" in the Host +// header and to http://example.net/ (default TCP port 80) with "example.net:2" +// in the Host header, verifying that the egress gateway dials the TCP port the +// actor connected to rather than the port from the Host header. +// Sending 2 requests per target verifies that cached policy correctly sets the +// destination port filter state. +func TestActorEgressDialedPortOverridesHostHeader(t *testing.T) { + ctx := context.Background() + router, actorRef := hostnamePolicyActor(t, ctx) + + for _, tc := range []struct { + url string + host string + }{ + {url: "https://example.com/", host: "example.com:1"}, + {url: "http://example.net/", host: "example.net:2"}, + } { + payload := []byte(fmt.Sprintf(`{"url":%q,"host":%q,"disableKeepAlive":true}`, tc.url, tc.host)) + for i := range 2 { + status, body := postThroughEgressActorUntil(t, ctx, router, actorRef, "/", payload, reached) + if status != http.StatusOK { + t.Fatalf("request %d to %s with Host %s returned HTTP %d, want 200; body: %s", i+1, tc.url, tc.host, status, body) + } + } + } +} + // fetchThroughEgressActorUntil is fetchThroughEgressActor with the caller // deciding which answer is final. func fetchThroughEgressActorUntil(t *testing.T, ctx context.Context, router *e2e.RouterClient, actorRef resources.ActorRef, url string, done func(status int, body []byte) bool) (int, []byte) { From 56f44c1dc57b0adab8450998cf25d948a833c58b Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 00:44:58 +0000 Subject: [PATCH 16/23] Update doc Signed-off-by: Yan Avlasov --- .../envoy/dynamic-modules/egress-policy-cache/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md index 4faceef20c..f0736b439e 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -36,6 +36,8 @@ When an actor opens a `CONNECT` tunnel on the outer listener: `dev.ate.policy.egress.cached`. - Sets `has_cached_policy = true` on the per-stream `EgressPolicyCacheFilter`. + - Sets `dev.ate.egress:dialed_port` in the dynamic metadata, so that inner + request is sent to the port in the CONNECT authority. - Increments the `ate_egress.connect_cache_hit` counter. - **Expired entry (`stored_at.elapsed() > cache_ttl`)**: - Removes the expired entry from the LRU cache without setting filter From caeb1f75cc18197146ddb931821fff032d3275d4 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 01:31:15 +0000 Subject: [PATCH 17/23] Use dedicated value to skip ext_proc Signed-off-by: Yan Avlasov --- .../egress-policy-cache/README.md | 13 ++++++------- .../egress-policy-cache/src/lib.rs | 19 +++++++++++++++++++ manifests/ate-install/atenet-egress.yaml | 10 ++++------ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md index f0736b439e..2ad1c52819 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -34,6 +34,7 @@ When an actor opens a `CONNECT` tunnel on the outer listener: - **Fresh hit (`stored_at.elapsed() <= cache_ttl`)**: - Writes the cached policy JSON to filter state under key `dev.ate.policy.egress.cached`. + - Sets filter state key `dev.ate.policy.egress.skip_callout` to `"true"`. - Sets `has_cached_policy = true` on the per-stream `EgressPolicyCacheFilter`. - Sets `dev.ate.egress:dialed_port` in the dynamic metadata, so that inner @@ -46,9 +47,9 @@ When an actor opens a `CONNECT` tunnel on the outer listener: - **Cache miss (or missing certificate digest / destination port)**: - Increments the `ate_egress.connect_cache_miss` counter. 2. Subsequent filters in the outer HTTP filter chain act on the result: - - `envoy.filters.http.composite` checks for the presence of - `dev.ate.policy.egress.cached` in filter state and invokes - `envoy.filters.http.ext_proc` only when it is absent. + - `envoy.filters.http.composite` checks whether + `dev.ate.policy.egress.skip_callout` equals `"true"` in filter state and + invokes `envoy.filters.http.ext_proc` only when it does not. - `envoy.filters.http.set_filter_state` copies either `%DYNAMIC_METADATA(dev.ate.policy.egress)%` (on a cache miss, populated by `ext_proc`) or `%FILTER_STATE(dev.ate.policy.egress.cached:PLAIN)%` (on a @@ -162,11 +163,9 @@ filter before the conditional `ext_proc` composite filter and the post-`ext_proc name: filter_state typed_config: "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.FilterStateInput - key: dev.ate.policy.egress.cached + key: dev.ate.policy.egress.skip_callout value_match: - safe_regex: - google_re2: {} - regex: ".*" + exact: "true" on_match: action: name: composite-action diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs index 4d08d425b7..8a902916cc 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/src/lib.rs @@ -44,6 +44,14 @@ pub const ATE_POLICY_EGRESS: &str = "dev.ate.policy.egress"; // state directly and share it with upstream. pub const ATE_POLICY_EGRESS_CACHED: &str = "dev.ate.policy.egress.cached"; +/// Filter state key set to `"true"` on a cache hit so the composite filter can +/// skip the `ext_proc` callout. +pub const ATE_POLICY_EGRESS_SKIP_CALLOUT: &str = "dev.ate.policy.egress.skip_callout"; + +/// Filter state value for [`ATE_POLICY_EGRESS_SKIP_CALLOUT`] when the `ext_proc` +/// callout should be skipped. +pub const SKIP_CALLOUT_TRUE: &str = "true"; + /// Dynamic metadata namespace for egress attributes set on CONNECT. pub const ATE_EGRESS_METADATA_NAMESPACE: &str = "dev.ate.egress"; @@ -301,6 +309,10 @@ impl HttpFilter for EgressPolicyCacheFilter { if let Some((policy, port)) = cached_policy { self.has_cached_policy = true; envoy_filter.set_filter_state_bytes(ATE_POLICY_EGRESS_CACHED.as_bytes(), policy.as_bytes()); + envoy_filter.set_filter_state_bytes( + ATE_POLICY_EGRESS_SKIP_CALLOUT.as_bytes(), + SKIP_CALLOUT_TRUE.as_bytes(), + ); envoy_filter.set_dynamic_metadata_string( ATE_EGRESS_METADATA_NAMESPACE, ATE_EGRESS_DIALED_PORT_KEY, @@ -645,6 +657,13 @@ mod tests { }) .return_const(true) .once(); + mock_filter + .expect_set_filter_state_bytes() + .withf(|key, val| { + key == ATE_POLICY_EGRESS_SKIP_CALLOUT.as_bytes() && val == SKIP_CALLOUT_TRUE.as_bytes() + }) + .return_const(true) + .once(); mock_filter .expect_set_dynamic_metadata_string() .withf(|namespace, key, val| { diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 1fe96279ed..e07fd7f478 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -187,8 +187,8 @@ data: # certificate (see cmd/atenet/internal/router/egress) and returns # the SNI rules in dev.ate.policy.egress and the dialed port in # dev.ate.egress. Fails closed if the router is down. - # Skipped when envoy_substrate_egress_policy_cache already populated - # dev.ate.policy.egress.cached in filter state on a cache hit. + # Skipped when envoy_substrate_egress_policy_cache already set + # dev.ate.policy.egress.skip_callout to "true" in filter state on a cache hit. - name: envoy.filters.http.composite typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite @@ -202,11 +202,9 @@ data: name: filter_state typed_config: "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.FilterStateInput - key: dev.ate.policy.egress.cached + key: dev.ate.policy.egress.skip_callout value_match: - safe_regex: - google_re2: {} - regex: ".*" + exact: "true" on_match: action: name: composite-action From 2839ea6a823e471497592dd0f8a3008a4acec2d2 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 01:46:08 +0000 Subject: [PATCH 18/23] Massage doc Signed-off-by: Yan Avlasov --- .../egress-policy-cache/README.md | 83 +++---------------- 1 file changed, 10 insertions(+), 73 deletions(-) diff --git a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md index 2ad1c52819..13cb110bad 100644 --- a/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md +++ b/cmd/dataplane/envoy/dynamic-modules/egress-policy-cache/README.md @@ -22,25 +22,26 @@ policy JSON string alongside the `Instant` (`stored_at`) when it was cached, keyed by `;`, where `` is the downstream peer certificate's SHA-256 digest (`connection.sha256_peer_certificate_digest`) and `` is the -destination port extracted from the `:authority` request header. +destination port extracted from the CONNECT `:authority` request header. ### Request path (`on_request_headers`) When an actor opens a `CONNECT` tunnel on the outer listener: 1. The filter builds the cache key `;` from - `connection.sha256_peer_certificate_digest` and the `:authority` request - header, and looks it up in the current worker thread's LRU cache: - - **Fresh hit (`stored_at.elapsed() <= cache_ttl`)**: + `connection.sha256_peer_certificate_digest` and the port from the CONNECT `:authority` + header, and looks up cached policy in worker thread local LRU cache: + - **If unexpired entry was found.**: - Writes the cached policy JSON to filter state under key `dev.ate.policy.egress.cached`. - - Sets filter state key `dev.ate.policy.egress.skip_callout` to `"true"`. + - Sets filter state key `dev.ate.policy.egress.skip_callout` to `"true"` to skip the ext_proc + callout. - Sets `has_cached_policy = true` on the per-stream `EgressPolicyCacheFilter`. - Sets `dev.ate.egress:dialed_port` in the dynamic metadata, so that inner request is sent to the port in the CONNECT authority. - Increments the `ate_egress.connect_cache_hit` counter. - - **Expired entry (`stored_at.elapsed() > cache_ttl`)**: + - **If expired entry (`stored_at.elapsed() > cache_ttl`) was found**: - Removes the expired entry from the LRU cache without setting filter state. - Increments the `ate_egress.connect_cache_miss` counter. @@ -62,13 +63,10 @@ When an actor opens a `CONNECT` tunnel on the outer listener: When the `CONNECT` response headers arrive: 1. If `cache_enabled` is `false`, the filter returns `Continue` immediately. -2. If `:status` is `200` and `!self.has_cached_policy`: - - Reads the egress policy JSON from filter state key - `dev.ate.policy.egress` and builds the cache key - `;` from - `connection.sha256_peer_certificate_digest` and `:authority`. +2. If `:status` is `200` and `self.has_cached_policy` is `false`: + - Builds cache value from the `dev.ate.policy.egress` filer state. - Inserts `CachedPolicy { policy, stored_at: Instant::now() }` into the - current worker thread's LRU cache. + current worker thread's LRU cache at the the `;` key. 3. If `self.has_cached_policy` is `true` (the policy for this stream came from the cache), caching is skipped so the entry's original `stored_at` timestamp is preserved until `cache_ttl` expires. @@ -137,65 +135,4 @@ hack/test-dynamic-modules.sh > worker thread (`--concurrency 1`) so consecutive `CONNECT` requests are > routed to the same thread-local cache instance. -## Envoy configuration - -In the outer `CONNECT` listener's `http_filters`, place the dynamic module -filter before the conditional `ext_proc` composite filter and the post-`ext_proc` -`set_filter_state` filter: - -```yaml -- name: envoy.filters.http.dynamic_modules - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_modules.v3.DynamicModuleFilter - dynamic_module_config: - name: envoy_substrate_egress_policy_cache - filter_name: envoy_substrate_egress_policy_cache -- name: envoy.filters.http.composite - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite - matcher: - matcher_list: - matchers: - - predicate: - not_matcher: - single_predicate: - input: - name: filter_state - typed_config: - "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.FilterStateInput - key: dev.ate.policy.egress.skip_callout - value_match: - exact: "true" - on_match: - action: - name: composite-action - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.ExecuteFilterAction - typed_config: - name: envoy.filters.http.ext_proc - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - # ... -- name: envoy.filters.http.set_filter_state - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config - on_request_headers: - - object_key: dev.ate.policy.egress - factory_key: envoy.string - skip_if_empty: true - shared_with_upstream: ONCE - format_string: - text_format_source: - inline_string: "%DYNAMIC_METADATA(dev.ate.policy.egress)%" - omit_empty_values: true - - object_key: dev.ate.policy.egress - factory_key: envoy.string - skip_if_empty: true - shared_with_upstream: ONCE - format_string: - text_format_source: - inline_string: "%FILTER_STATE(dev.ate.policy.egress.cached:PLAIN)%" - omit_empty_values: true -``` - See `manifests/ate-install/atenet-egress.yaml` for the complete configuration. From 2275f436b44b6da42d433c0ea6d9a217ec4731a9 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 02:14:25 +0000 Subject: [PATCH 19/23] fix tests Signed-off-by: Yan Avlasov --- internal/e2e/collector_metrics.go | 3 +++ internal/e2e/suites/egressmitm/egressmitm_test.go | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/e2e/collector_metrics.go b/internal/e2e/collector_metrics.go index 9b0b392ed6..7044227580 100644 --- a/internal/e2e/collector_metrics.go +++ b/internal/e2e/collector_metrics.go @@ -46,6 +46,9 @@ const ( // EgressExtProcPolicyMITMStatPrefix is the Envoy ext_proc filter stat_prefix // on the inner MITM HTTP leg in atenet-egress. EgressExtProcPolicyMITMStatPrefix = "egress_policy_mitm" + // EgressExtProcPolicyClearTextStatPrefix is the Envoy ext_proc filter stat_prefix + // on the inner clear text HTTP leg in atenet-egress. + EgressExtProcPolicyClearTextStatPrefix = "egress_policy_cleartext" // EgressConnectCacheHitCounter is the Envoy dynamic-module counter name for // CONNECT policy cache hits in atenet-egress. diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 6cf66b1e84..2a50b34724 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -201,13 +201,13 @@ func TestActorEgressMITMTrust(t *testing.T) { } afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) - // Each of the 6 fetches opens a CONNECT tunnel (egress_identity), while - // only the positive MITM fetch completes the inner TLS handshake and sends - // an HTTP request through the MITM chain (egress_policy_mitm). - // Since caching is enabled only the first CONNECT request will result in ext_proc call. + // All connections go either to port 80 or port 443. Since policy is cached + // per actor per destination port, expect 2 ext_proc callouts at the CONNECT + // filter chain, 2 at the MITM filter chain and 1 in clear text chain. const ( - wantIdentityCalls = 1 - wantMITMPolicyCalls = 1 + wantIdentityCalls = 2 + wantMITMPolicyCalls = 2 + wantClearTextPolicyCalls = 1 ) if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantIdentityCalls { t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcIdentityStatPrefix, got, wantIdentityCalls) @@ -215,6 +215,9 @@ func TestActorEgressMITMTrust(t *testing.T) { if got := afterExtProc[e2e.EgressExtProcPolicyMITMStatPrefix] - beforeExtProc[e2e.EgressExtProcPolicyMITMStatPrefix]; got != wantMITMPolicyCalls { t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcPolicyMITMStatPrefix, got, wantMITMPolicyCalls) } + if got := afterExtProc[e2e.EgressExtProcPolicyClearTextStatPrefix] - beforeExtProc[e2e.EgressExtProcPolicyClearTextStatPrefix]; got != wantClearTextPolicyCalls { + t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcPolicyClearTextStatPrefix, got, wantClearTextPolicyCalls) + } } // egressOriginHost is the one host the probe actor's EgressPolicy allows. From 74eb937dbd8e838cd7090fddef89c97baaed50f9 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 02:40:34 +0000 Subject: [PATCH 20/23] Revert egressmitm_test.go Signed-off-by: Yan Avlasov --- .../e2e/suites/egressmitm/egressmitm_test.go | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/internal/e2e/suites/egressmitm/egressmitm_test.go b/internal/e2e/suites/egressmitm/egressmitm_test.go index 2a50b34724..5b0876fb15 100644 --- a/internal/e2e/suites/egressmitm/egressmitm_test.go +++ b/internal/e2e/suites/egressmitm/egressmitm_test.go @@ -99,17 +99,8 @@ func TestActorEgressMITMTrust(t *testing.T) { // deleted it), leaving the gateway briefly signing with the old CA — so // certificate failures retry for one propagation window before counting. deadline := time.Now().Add(2 * time.Minute) - var ( - beforeExtProc map[string]int - pos fetchResponse - ) + var pos fetchResponse for { - beforeScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) - if err != nil { - t.Fatalf("ScrapeEgressEnvoyMetrics before fetches: %v", err) - } - beforeExtProc = e2e.EgressExtProcStreamCounts(beforeScrape) - pos = probeFetch(t, ctx, rc, id, origin, "bundle") isCertErr := strings.Contains(pos.Error, "certificate") || strings.Contains(pos.Error, "x509") if pos.Error == "" || !isCertErr || time.Now().After(deadline) { @@ -189,35 +180,6 @@ func TestActorEgressMITMTrust(t *testing.T) { } }) } - - if !e2e.CurrentAtenetDataplane().IsDataplaneConcurrencyDisabled() { - t.Log("skipping counter verification: dataplane concurrency is not disabled") - return - } - - afterScrape, err := e2e.ScrapeEgressEnvoyMetrics(ctx) - if err != nil { - t.Fatalf("ScrapeEgressEnvoyMetrics after fetches: %v", err) - } - afterExtProc := e2e.EgressExtProcStreamCounts(afterScrape) - - // All connections go either to port 80 or port 443. Since policy is cached - // per actor per destination port, expect 2 ext_proc callouts at the CONNECT - // filter chain, 2 at the MITM filter chain and 1 in clear text chain. - const ( - wantIdentityCalls = 2 - wantMITMPolicyCalls = 2 - wantClearTextPolicyCalls = 1 - ) - if got := afterExtProc[e2e.EgressExtProcIdentityStatPrefix] - beforeExtProc[e2e.EgressExtProcIdentityStatPrefix]; got != wantIdentityCalls { - t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcIdentityStatPrefix, got, wantIdentityCalls) - } - if got := afterExtProc[e2e.EgressExtProcPolicyMITMStatPrefix] - beforeExtProc[e2e.EgressExtProcPolicyMITMStatPrefix]; got != wantMITMPolicyCalls { - t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcPolicyMITMStatPrefix, got, wantMITMPolicyCalls) - } - if got := afterExtProc[e2e.EgressExtProcPolicyClearTextStatPrefix] - beforeExtProc[e2e.EgressExtProcPolicyClearTextStatPrefix]; got != wantClearTextPolicyCalls { - t.Errorf("egress ext_proc %s streams_started delta = %d, want %d", e2e.EgressExtProcPolicyClearTextStatPrefix, got, wantClearTextPolicyCalls) - } } // egressOriginHost is the one host the probe actor's EgressPolicy allows. From 2bcd4cabfba17183b2a03bc41295a795b87bf783 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 16:09:56 +0000 Subject: [PATCH 21/23] Make stats flushing immediate in E2E tests Signed-off-by: Yan Avlasov --- cmd/ate-setup/internal/steps/overlay.go | 19 ++++++++++++++++--- cmd/ate-setup/internal/steps/overlay_test.go | 17 ++++++++++++----- manifests/ate-install/atenet-egress.yaml | 3 +++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 620cc597f4..a3faf78e46 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -192,17 +192,30 @@ func (e *Env) patchEnvoyDataplaneImage(raw []byte, imageRef string) []byte { // patchEnvoyConcurrency replaces the - ${E2E_ENVOY_CONCURRENCY} placeholder in // the manifest with --concurrency when concurrency is set, or removes -// the placeholder line when empty. +// the placeholder line when empty. It also replaces the +// stats_flush_on_admin: ${E2E_ENVOY_STATS_FLUSH_ON_ADMIN} placeholder in the +// Envoy bootstrap config with stats_flush_on_admin: true when concurrency is +// "1", or removes the placeholder line otherwise. func (e *Env) patchEnvoyConcurrency(raw []byte, concurrency string) []byte { - const placeholder = "- ${E2E_ENVOY_CONCURRENCY}" + const ( + concurrencyPlaceholder = "- ${E2E_ENVOY_CONCURRENCY}" + statsFlushPlaceholder = "stats_flush_on_admin: ${E2E_ENVOY_STATS_FLUSH_ON_ADMIN}" + ) var out []string for _, line := range strings.Split(string(raw), "\n") { - if strings.TrimSpace(line) == placeholder { + switch strings.TrimSpace(line) { + case concurrencyPlaceholder: if concurrency != "" { indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] out = append(out, indent+"- --concurrency", fmt.Sprintf("%s- %q", indent, concurrency)) } continue + case statsFlushPlaceholder: + if concurrency == "1" { + indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + out = append(out, indent+"stats_flush_on_admin: true") + } + continue } out = append(out, line) } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 73e2d48305..32f64d8aa6 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -817,7 +817,7 @@ func TestRenderAtenetEgressManifestPrebuilt(t *testing.T) { if !strings.Contains(string(out), "image: "+want) { t.Errorf("rendered manifest does not install %s:\n%s", want, out) } - for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "${E2E_ENVOY_CONCURRENCY}", "ko://"} { + for _, leftover := range []string{"${ENVOY_DATAPLANE_IMAGE}", "${E2E_ENVOY_CONCURRENCY}", "${E2E_ENVOY_STATS_FLUSH_ON_ADMIN}", "ko://"} { if strings.Contains(string(out), leftover) { t.Errorf("rendered manifest still contains %q", leftover) } @@ -865,17 +865,24 @@ func TestPatchEnvoyDataplaneImage(t *testing.T) { func TestPatchEnvoyConcurrency(t *testing.T) { e := &Env{} - raw := []byte("containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - ${E2E_ENVOY_CONCURRENCY}\n") + raw := []byte("data:\n envoy.yaml: |\n stats_flush_on_admin: ${E2E_ENVOY_STATS_FLUSH_ON_ADMIN}\ncontainers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - ${E2E_ENVOY_CONCURRENCY}\n") - t.Run("set", func(t *testing.T) { - want := "containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - --concurrency\n - \"1\"\n" + t.Run("set to 1", func(t *testing.T) { + want := "data:\n envoy.yaml: |\n stats_flush_on_admin: true\ncontainers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - --concurrency\n - \"1\"\n" if got := string(e.patchEnvoyConcurrency(raw, "1")); got != want { t.Errorf("patchEnvoyConcurrency(1) = %q, want %q", got, want) } }) + t.Run("set to other value", func(t *testing.T) { + want := "data:\n envoy.yaml: |\ncontainers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n - --concurrency\n - \"2\"\n" + if got := string(e.patchEnvoyConcurrency(raw, "2")); got != want { + t.Errorf("patchEnvoyConcurrency(2) = %q, want %q", got, want) + } + }) + t.Run("unset", func(t *testing.T) { - want := "containers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n" + want := "data:\n envoy.yaml: |\ncontainers:\n- name: envoy\n args:\n - -c\n - /etc/envoy/envoy.yaml\n" if got := string(e.patchEnvoyConcurrency(raw, "")); got != want { t.Errorf("patchEnvoyConcurrency(\"\") = %q, want %q", got, want) } diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index e07fd7f478..6365819bff 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -56,6 +56,9 @@ data: # dialling the pod IP, which is IPv4 on an IPv4 cluster. socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } + # E2E_ENVOY_STATS_FLUSH_ON_ADMIN is substituted in renderAtenetEgressManifest + stats_flush_on_admin: ${E2E_ENVOY_STATS_FLUSH_ON_ADMIN} + # Required by the mitm_listener below. An internal listener has no socket, # so it is reachable only through a cluster with an envoy_internal_address; # without this extension registered, that listener silently fails to load. From cf762240fbd3787c9d83be0c622e7720c3e67d98 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Wed, 7 Oct 2026 20:18:11 +0000 Subject: [PATCH 22/23] Update doc Signed-off-by: Yan Avlasov --- docs/operator-install.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/operator-install.md b/docs/operator-install.md index 902382a6d1..b1f6ec8a0c 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -286,6 +286,7 @@ command. A setting with no default is unset unless something supplies it. | `demo.counter.storageClass` | `STORAGE_CLASS` | `--storage-class` | `standard` | `deploy demo counter` | | `demo.counter.withExternalVolume` | `ATE_DEMO_COUNTER_WITH_EXTERNAL_VOLUME` | `--with-external-volume` | `false` | `deploy demo counter` | | `docker.buildFlags` | `DOCKER_BUILD_FLAGS` | `--docker-build-flags` | — | (global) | +| `e2e.envoyConcurrency` | `E2E_ENVOY_CONCURRENCY` | `--e2e-envoy-concurrency` | — | (global) | | `gcp.clusterLocation` | `CLUSTER_LOCATION` | `--gcp-cluster-location` | — | (global) | | `gcp.clusterName` | `CLUSTER_NAME` | `--gcp-cluster-name` | — | (global) | | `gcp.projectID` | `PROJECT_ID` | `--gcp-project-id` | — | (global) | From 00cea33ead52dddd8bfb229e3b58470a9c4cbf10 Mon Sep 17 00:00:00 2001 From: Yan Avlasov Date: Thu, 8 Oct 2026 02:13:16 +0000 Subject: [PATCH 23/23] Update comment Signed-off-by: Yan Avlasov --- internal/e2e/suites/networking/egresspolicy_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/e2e/suites/networking/egresspolicy_test.go b/internal/e2e/suites/networking/egresspolicy_test.go index 6dc7c7e528..a214084953 100644 --- a/internal/e2e/suites/networking/egresspolicy_test.go +++ b/internal/e2e/suites/networking/egresspolicy_test.go @@ -209,8 +209,7 @@ func TestActorEgressPolicyCache(t *testing.T) { } // TestActorEgressPolicyCacheExpiration verifies that cached egress policy -// entries expire after the configured cache_ttl (5s default), triggering a -// cache miss and a new ext_proc lookup on the next request. +// entries expire after the configured cache_ttl. func TestActorEgressPolicyCacheExpiration(t *testing.T) { ctx := context.Background() router, actorRef := hostnamePolicyActor(t, ctx)