Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions ratelimiter/method_bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import "strings"
const (
// PlaneCometBFT is the rate-limit plane label for Tendermint RPC HTTP.
PlaneCometBFT = "cometbft"
// PlaneGRPC is the rate-limit plane label for native gRPC (:9090).
PlaneGRPC = "grpc"

// rpcMethodBucketOther is the fallback label for unrecognized methods.
rpcMethodBucketOther = "other"
Expand Down Expand Up @@ -75,6 +77,37 @@ var knownCometBFTRPCMethods = map[string]struct{}{
"websocket": {},
}

// knownGRPCServices lists protobuf service names registered on the native gRPC
// server. Rejection metrics on PlaneGRPC record the service name rather than
// the full /service/Method path, keeping OTel attribute cardinality bounded.
var knownGRPCServices = map[string]struct{}{
"cosmos.auth.v1beta1.Query": {},
"cosmos.authz.v1beta1.Query": {},
"cosmos.bank.v1beta1.Query": {},
"cosmos.base.reflection.v2alpha1.ReflectionService": {},
"cosmos.base.tendermint.v1beta1.Service": {},
"cosmos.consensus.v1.Query": {},
"cosmos.distribution.v1beta1.Query": {},
"cosmos.evidence.v1beta1.Query": {},
"cosmos.feegrant.v1beta1.Query": {},
"cosmos.gov.v1beta1.Query": {},
"cosmos.mint.v1beta1.Query": {},
"cosmos.params.v1beta1.Query": {},
"cosmos.slashing.v1beta1.Query": {},
"cosmos.staking.v1beta1.Query": {},
"cosmos.tx.v1beta1.Service": {},
"cosmos.upgrade.v1beta1.Query": {},
"cosmos.vesting.v1beta1.Query": {},
"grpc.reflection.v1alpha.ServerReflection": {},
"seiprotocol.seichain.epoch.Query": {},
"seiprotocol.seichain.evm.Query": {},
"seiprotocol.seichain.mint.Query": {},
"seiprotocol.seichain.oracle.Query": {},
"seiprotocol.seichain.tokenfactory.Query": {},
"cosmos.circuit.v1.Query": {},
"cosmwasm.wasm.v1.Query": {},
}

// bucketRPCMethod maps a raw JSON-RPC method name to a low-cardinality label
// suitable for OTel/Prometheus metrics. Attacker-controlled method strings
// collapse to rpcMethodBucketOther.
Expand All @@ -85,9 +118,35 @@ func bucketRPCMethod(plane, method string) string {
if plane == PlaneCometBFT {
return bucketCometBFTRPCMethod(method)
}
if plane == PlaneGRPC {
return bucketGRPCMethod(method)
}
return bucketNamespacedRPCMethod(method)
}

func bucketGRPCMethod(fullMethod string) string {
if fullMethod == "" || len(fullMethod) > maxRPCMethodLen {
return rpcMethodBucketOther
}
svc := grpcServiceFromFullMethod(fullMethod)
if svc == "" || len(svc) > maxRPCMethodLen {
return rpcMethodBucketOther
}
if _, ok := knownGRPCServices[svc]; ok {
return svc
}
return rpcMethodBucketOther
}

func grpcServiceFromFullMethod(fullMethod string) string {
method := strings.TrimPrefix(fullMethod, "/")
slash := strings.LastIndexByte(method, '/')
if slash <= 0 {
return ""
}
return method[:slash]
}

func bucketCometBFTRPCMethod(method string) string {
if method == "" || len(method) > maxRPCMethodLen {
return rpcMethodBucketOther
Expand Down
13 changes: 13 additions & 0 deletions ratelimiter/method_bucket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,17 @@ func TestBucketRPCMethod_CometBFTUnknownMethods(t *testing.T) {
func TestBucketRPCMethod_Invalid(t *testing.T) {
require.Equal(t, MethodInvalid, bucketRPCMethod("evm", MethodInvalid))
require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneCometBFT, MethodInvalid))
require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneGRPC, MethodInvalid))
}

func TestBucketRPCMethod_GrpcknownServices(t *testing.T) {
require.Equal(t, "cosmos.bank.v1beta1.Query", bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance"))
require.Equal(t, "cosmos.tx.v1beta1.Service", bucketRPCMethod(PlaneGRPC, "/cosmos.tx.v1beta1.Service/Simulate"))
}

func TestBucketRPCMethod_GrpcUnknownServices(t *testing.T) {
require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, ""))
require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/bogus.Service/Call"))
require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "not-a-grpc-path"))
require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, strings.Repeat("a", maxRPCMethodLen+1)))
}
49 changes: 49 additions & 0 deletions sei-cosmos/server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"
"time"

"github.com/sei-protocol/sei-chain/ratelimiter"
storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
"github.com/sei-protocol/sei-chain/sei-cosmos/telemetry"
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
Expand Down Expand Up @@ -258,6 +259,33 @@ type GRPCConfig struct {
// KeepalivePermitWithoutStream defines whether the server allows keepalive
// pings even when there are no active streams.
KeepalivePermitWithoutStream bool `mapstructure:"keepalive-permit-without-stream"`

// IPRateLimitRPS is the per-IP sustained request rate in requests/second for
// native gRPC (:9090). Zero disables the token bucket (Allow always returns
// true) and does not bypass the admission interceptor when
// rate-limiting-enabled is true.
IPRateLimitRPS float64 `mapstructure:"ip-rate-limit-rps"`

// IPRateLimitBurst is the maximum per-IP burst size. Zero disables the token
// bucket (same effect as ip-rate-limit-rps = 0) and does not bypass the
// admission interceptor when rate-limiting-enabled is true.
IPRateLimitBurst int `mapstructure:"ip-rate-limit-burst"`

// RateLimitingEnabled is the master switch for gRPC rate-limit admission.
RateLimitingEnabled bool `mapstructure:"rate-limiting-enabled"`

// TrustedProxyCIDRs lists CIDRs whose x-forwarded-for metadata is trusted
// when resolving the client IP for rate limiting. Empty means trust no proxy.
TrustedProxyCIDRs []string `mapstructure:"trusted-proxy-cidrs"`
}

// RateLimiterConfig builds the ratelimiter.Config used by native gRPC admission.
func (c GRPCConfig) RateLimiterConfig() ratelimiter.Config {
return ratelimiter.Config{
RPS: c.IPRateLimitRPS,
Burst: c.IPRateLimitBurst,
TrustedProxyCIDRs: c.TrustedProxyCIDRs,
}
}

// GRPCWebConfig defines configuration for the gRPC-web server.
Expand Down Expand Up @@ -386,6 +414,10 @@ func DefaultConfig() *Config {
KeepaliveTimeout: DefaultGRPCKeepaliveTimeout,
KeepaliveMinTime: DefaultGRPCKeepaliveMinTime,
KeepalivePermitWithoutStream: DefaultGRPCKeepalivePermitWithoutStream,
IPRateLimitRPS: ratelimiter.DefaultRPS,
IPRateLimitBurst: ratelimiter.DefaultBurst,
RateLimitingEnabled: false,
TrustedProxyCIDRs: nil,
},
Rosetta: RosettaConfig{
Enable: false,
Expand Down Expand Up @@ -572,6 +604,19 @@ func GetConfig(v *viper.Viper) (Config, error) {
grpcMaxConnectionAge := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age"), DefaultGRPCMaxConnectionAge)
grpcMaxConnectionAgeGrace := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age-grace"), DefaultGRPCMaxConnectionAgeGrace)

grpcIPRateLimitRPS := ratelimiter.DefaultRPS
if v.IsSet("grpc.ip-rate-limit-rps") {
grpcIPRateLimitRPS = v.GetFloat64("grpc.ip-rate-limit-rps")
}
grpcIPRateLimitBurst := ratelimiter.DefaultBurst
if v.IsSet("grpc.ip-rate-limit-burst") {
grpcIPRateLimitBurst = v.GetInt("grpc.ip-rate-limit-burst")
}
grpcTrustedProxyCIDRs := []string(nil)
if v.IsSet("grpc.trusted-proxy-cidrs") {
grpcTrustedProxyCIDRs = v.GetStringSlice("grpc.trusted-proxy-cidrs")
}

cfg := Config{
BaseConfig: BaseConfig{
MinGasPrices: v.GetString("minimum-gas-prices"),
Expand Down Expand Up @@ -627,6 +672,10 @@ func GetConfig(v *viper.Viper) (Config, error) {
KeepaliveTimeout: grpcKeepaliveTimeout,
KeepaliveMinTime: grpcKeepaliveMinTime,
KeepalivePermitWithoutStream: v.GetBool("grpc.keepalive-permit-without-stream"),
IPRateLimitRPS: grpcIPRateLimitRPS,
IPRateLimitBurst: grpcIPRateLimitBurst,
RateLimitingEnabled: v.GetBool("grpc.rate-limiting-enabled"),
TrustedProxyCIDRs: grpcTrustedProxyCIDRs,
},
GRPCWeb: GRPCWebConfig{
Enable: v.GetBool("grpc-web.enable"),
Expand Down
29 changes: 26 additions & 3 deletions sei-cosmos/server/config/config_fuzz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1166,6 +1166,10 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) {
"grpc.keepalive-permit-without-stream",
cfg.GRPC.KeepalivePermitWithoutStream, def.GRPC.KeepalivePermitWithoutStream, false,
},
{
"grpc.rate-limiting-enabled",
cfg.GRPC.RateLimitingEnabled, def.GRPC.RateLimitingEnabled, false,
},
{"telemetry.service-name", cfg.Telemetry.ServiceName, def.Telemetry.ServiceName, false},
{"telemetry.enable-hostname", cfg.Telemetry.EnableHostname, def.Telemetry.EnableHostname, false},
{
Expand Down Expand Up @@ -1354,7 +1358,7 @@ func TestBaseConfigManifestNamesEveryField(t *testing.T) {
)
}

// grpcKeys covers the three [grpc] keys read as plain casts.
// grpcKeys covers the four [grpc] keys read as plain casts.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] The summary line was updated to "four" but the body below it still carries the old counts: line 1363-1364 says "only three keys are rows. Eight others are read behind v.IsSet", which is now four rows and eleven others. grpcKeysWithTargetsOfTheirOwn's godoc (line 1398) has the same drift — "Six are read behind v.IsSet ... The other two" is now nine and two, since ip-rate-limit-rps, ip-rate-limit-burst, and trusted-proxy-cidrs are all v.IsSet-guarded.

This suite is the repo's record of how configuration resolves, and its counts are what a reader checks the tables against, so the stale numbers actively mislead.

//
// The section is where the guarding in this reader is most complete, which is why only three keys
// are rows. Eight others are read behind v.IsSet or through clampNonNegativeDuration and so resolve
Expand All @@ -1380,6 +1384,12 @@ var grpcKeys = []configtest.KeySpec{
Why: "whether a client may ping with no active stream; false either way, so this row states " +
"the key is read rather than recording a divergence",
},
{
Key: "grpc.rate-limiting-enabled", Path: "RateLimitingEnabled",
Cast: configtest.CastBool, Unguarded: true,
Why: "the declared default is false and an absent key resolves false, so this row states " +
"the key is read rather than recording a divergence",
},
}

// grpcKeysWithTargetsOfTheirOwn are the [grpc] keys whose resolution a row cannot describe, recorded
Expand All @@ -1406,6 +1416,9 @@ var grpcKeysWithTargetsOfTheirOwn = []configtest.KeyName{
"grpc.keepalive-time",
"grpc.keepalive-timeout",
"grpc.keepalive-min-time",
"grpc.ip-rate-limit-rps",
"grpc.ip-rate-limit-burst",
"grpc.trusted-proxy-cidrs",
}

func readGRPC(t testing.TB) func(configtest.AppOpts) (any, error) {
Expand All @@ -1419,6 +1432,7 @@ func FuzzGRPCConfig(f *testing.F) {
seeds.AddRow(uint(0), fuzzing.KindBool, "", int64(0), true)
seeds.AddRow(uint(1), fuzzing.KindString, "127.0.0.1:19090", int64(0), false)
seeds.AddRow(uint(2), fuzzing.KindBool, "", int64(0), true)
seeds.AddRow(uint(3), fuzzing.KindBool, "", int64(0), true)

configtest.CheckEveryRowHasADiscriminatingSeed(f, "grpc", readGRPC(f), grpcKeys, seeds,
grpcKeysWithTargetsOfTheirOwn...)
Expand All @@ -1429,8 +1443,8 @@ func FuzzGRPCConfig(f *testing.F) {
})
}

// TestGRPCKeyNamesMatchTheRecordedNames pins all eleven [grpc] key names, the three rows and the
// eight driven elsewhere.
// TestGRPCKeyNamesMatchTheRecordedNames pins all fifteen [grpc] key names, the four rows and the
// eleven driven elsewhere.
//
// The eight had no record before this, because their target carries a local struct rather than a
// KeySpec table, so nothing held their spelling. That is the gap this closes.
Expand All @@ -1450,6 +1464,9 @@ func TestGRPCManifestNamesEveryField(t *testing.T) {
"KeepaliveTime",
"KeepaliveTimeout",
"KeepaliveMinTime",
"IPRateLimitRPS",
"IPRateLimitBurst",
"TrustedProxyCIDRs",
)
}

Expand Down Expand Up @@ -1485,6 +1502,8 @@ func TestGetConfigGRPCAbsentReads(t *testing.T) {
{"grpc.keepalive-time", got.KeepaliveTime, def.KeepaliveTime},
{"grpc.keepalive-timeout", got.KeepaliveTimeout, def.KeepaliveTimeout},
{"grpc.keepalive-min-time", got.KeepaliveMinTime, def.KeepaliveMinTime},
{"grpc.ip-rate-limit-rps", got.IPRateLimitRPS, def.IPRateLimitRPS},
{"grpc.ip-rate-limit-burst", got.IPRateLimitBurst, def.IPRateLimitBurst},
} {
if c.absent != c.declared {
t.Errorf("an absent %s resolved to %v rather than the declared %v, so its v.IsSet guard "+
Expand All @@ -1493,6 +1512,10 @@ func TestGetConfigGRPCAbsentReads(t *testing.T) {
}
}

if got.TrustedProxyCIDRs != nil {
t.Errorf("an absent grpc.trusted-proxy-cidrs resolved to %v rather than nil", got.TrustedProxyCIDRs)
}

// Read unconditionally and clamped. Nothing guards these, so the assertion is on the coincidence
// itself: the declared default is the getter's zero, which is why an absent key looks correct.
for _, c := range []struct {
Expand Down
5 changes: 5 additions & 0 deletions sei-cosmos/server/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/spf13/viper"
"github.com/stretchr/testify/require"

"github.com/sei-protocol/sei-chain/ratelimiter"
storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config"
Expand Down Expand Up @@ -81,6 +82,10 @@ func TestDefaultGRPCConfig(t *testing.T) {
require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout)
require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime)
require.Equal(t, DefaultGRPCKeepalivePermitWithoutStream, cfg.GRPC.KeepalivePermitWithoutStream)
require.Equal(t, ratelimiter.DefaultRPS, cfg.GRPC.IPRateLimitRPS)
require.Equal(t, ratelimiter.DefaultBurst, cfg.GRPC.IPRateLimitBurst)
require.False(t, cfg.GRPC.RateLimitingEnabled)
require.Nil(t, cfg.GRPC.TrustedProxyCIDRs)
}

// seedViperWithDefaultConfig renders the default app config template and reads
Expand Down
4 changes: 4 additions & 0 deletions sei-cosmos/server/config/testdata/grpc.keys.golden
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"grpc.enable"
"grpc.address"
"grpc.keepalive-permit-without-stream"
"grpc.rate-limiting-enabled"
# keys with a target of their own
"grpc.max-recv-msg-size"
"grpc.max-open-connections"
Expand All @@ -10,3 +11,6 @@
"grpc.keepalive-time"
"grpc.keepalive-timeout"
"grpc.keepalive-min-time"
"grpc.ip-rate-limit-rps"
"grpc.ip-rate-limit-burst"
"grpc.trusted-proxy-cidrs"
4 changes: 4 additions & 0 deletions sei-cosmos/server/config/testdata/server_config.golden
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ GRPC.KeepaliveTime = time.Duration(2h0m0s)
GRPC.KeepaliveTimeout = time.Duration(20s)
GRPC.KeepaliveMinTime = time.Duration(5m0s)
GRPC.KeepalivePermitWithoutStream = bool(false)
GRPC.IPRateLimitRPS = float64(200)
GRPC.IPRateLimitBurst = int(400)
GRPC.RateLimitingEnabled = bool(false)
GRPC.TrustedProxyCIDRs = <nil-slice>
Rosetta.Address = string(":8080")
Rosetta.Blockchain = string("app")
Rosetta.Network = string("network")
Expand Down
15 changes: 15 additions & 0 deletions sei-cosmos/server/config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,21 @@ keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}"
# KeepalivePermitWithoutStream defines whether the server allows keepalive pings even when there are no active streams.
keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }}

# ip-rate-limit-rps is the per-IP sustained request rate in requests/second for native gRPC (:9090).
# Zero disables per-IP throttling; set rate-limiting-enabled = false for a full bypass.
ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }}

# ip-rate-limit-burst is the maximum per-IP burst above the sustained rate.
# Zero disables per-IP throttling (same effect as ip-rate-limit-rps = 0).
ip-rate-limit-burst = {{ .GRPC.IPRateLimitBurst }}

# rate-limiting-enabled is the master switch for gRPC rate-limit admission interceptors.
rate-limiting-enabled = {{ .GRPC.RateLimitingEnabled }}

# trusted-proxy-cidrs lists CIDRs whose x-forwarded-for metadata is trusted when
# resolving the client IP for rate limiting. Empty means trust no proxy.
trusted-proxy-cidrs = [{{- range $i, $c := .GRPC.TrustedProxyCIDRs }}{{- if $i }}, {{ end }}"{{ $c }}"{{- end }}]

###############################################################################
### gRPC Web Configuration (Auto-managed) ###
###############################################################################
Expand Down
39 changes: 39 additions & 0 deletions sei-cosmos/server/grpc/rate_limit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package grpc

import (
"context"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/sei-protocol/sei-chain/ratelimiter"
)

var errRateLimited = status.Error(codes.ResourceExhausted, "too many requests")

// UnaryRateLimitInterceptor returns a server interceptor that applies per-IP
// token-bucket rate limiting before the handler runs. registry must be non-nil.
func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
ip := registry.IPFromGRPCContext(ctx)
if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) {
return nil, errRateLimited
}
return handler(ctx, req)
}
}

// StreamRateLimitInterceptor returns a server interceptor that applies per-IP
// token-bucket rate limiting when a client stream is established. registry must
// be non-nil.
func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServerInterceptor {
return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
ctx := ss.Context()
ip := registry.IPFromGRPCContext(ctx)
if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) {
return errRateLimited
}
return handler(srv, ss)
}
}
Loading
Loading