bench 268: Solana reads with endpoint fallback (Chainstack, Alchemy, Helius)

harnesses/terminal-fill-quality/README.md

@@ -395,7 +395,7 @@ stats plus the last 400 samples (`method_version`, `min_priced`,
 
 | Var | Default | Meaning |
 |---|---|---|
-| `HELIUS_API_KEY` / `SOLANA_RPC` | public RPC | RPC endpoint for the reads |
+| `SOLANA_RPC` | public RPC | comma-separated endpoints for the Solana reads, tried in order (the next on a rate limit or a transport error); `HELIUS_API_KEY`, when set, appends Helius after them, the public node comes last. Production: Chainstack, Alchemy free, Helius |
 | `RPC_RPS` | `8` | pacing, calls per second |
 | `TICK_SECONDS` | `60` | sweep interval |
 | `DAILY_TARGET` | `400` | swaps read per terminal per day (random draw from the feed) |
@@ -409,7 +409,7 @@ stats plus the last 400 samples (`method_version`, `min_priced`,
 | `MIN_PRICED` | `50` | priced samples before a terminal is published |
 | `MIN_RANK` | `100` | priced samples before a terminal is ranked |
 | `MIN_TRADE_USD` | `2` | dust threshold |
-| `EVM_RPC_<CHAIN>` | unset | comma-separated endpoints tried before the public ones for that chain. Production: `EVM_RPC_BNB` = Alchemy's free BNB node (the public BSC nodes refuse `eth_getBalance` at the previous block, which native sells need; ~11 M compute units a month at 400 swaps a day per row); `EVM_RPC_ROBINHOOD` = the QuickNode Robinhood Chain endpoint (the public one has no WebSocket, rate-limits, and its `eth_getLogs` takes 9 s for 50 blocks or times out upstream; QuickNode serves 1,000 blocks of router logs in 0.8 s and past balances; the chain makes ~590 blocks a minute) |
+| `EVM_RPC_<CHAIN>` | unset | comma-separated endpoints tried before the public ones for that chain. Production: `EVM_RPC_BNB` = Alchemy's free BNB node (the public BSC nodes refuse `eth_getBalance` at the previous block, which native sells need; ~11 M compute units a month at 400 swaps a day per row); `EVM_RPC_ROBINHOOD` = the QuickNode Robinhood Chain endpoint then Chainstack's (QuickNode reads a 1,000-block log range in 0.13 s, Chainstack 1.3 s: a fallback, not a primary; Alchemy's free app does not serve the chain) (the public one has no WebSocket, rate-limits, and its `eth_getLogs` takes 9 s for 50 blocks or times out upstream; QuickNode serves 1,000 blocks of router logs in 0.8 s and past balances; the chain makes ~590 blocks a minute) |
 | `STATE_FILE` | unset | persist the window across restarts |
 | `HISTORY_FILE_PUBLIC` | unset | public JSON mirror |
 

harnesses/terminal-fill-quality/cmd/script/main.go

@@ -304,14 +304,24 @@ type Public struct {
 }
 
 func main() {
-	rpcURL := os.Getenv("SOLANA_RPC")
-	if rpcURL == "" {
-		if k := os.Getenv("HELIUS_API_KEY"); k != "" {
-			rpcURL = "https://mainnet.helius-rpc.com/?api-key=" + k
-		} else {
-			rpcURL = "https://api.mainnet-beta.solana.com"
-		}
-	}
+	// SOLANA_RPC: comma-separated endpoints tried in order (the next on a
+	// rate limit or a transport error); the Helius key, when set, joins the
+	// list after them, the public node last. Production: Chainstack's
+	// shared Solana node first (a paid node with no per-call budget), then
+	// Alchemy's free app for the overflow, then Helius (free tier, 1 M
+	// credits a month: the bench alone used 0.9 M when it was primary).
+	var rpcURLs []string
+	for _, u := range strings.Split(os.Getenv("SOLANA_RPC"), ",") {
+		if u = strings.TrimSpace(u); u != "" {
+			rpcURLs = append(rpcURLs, u)
+		}
+	}
+	if k := os.Getenv("HELIUS_API_KEY"); k != "" {
+		rpcURLs = append(rpcURLs, "https://mainnet.helius-rpc.com/?api-key="+k)
+	}
+	rpcURLs = append(rpcURLs, "https://api.mainnet-beta.solana.com")
+	rpcURL := rpcURLs[0]
+	log.Printf("solana reads: %d endpoint(s), primary %s", len(rpcURLs), redactURL(rpcURL, rpcURL))
 	tick := time.Duration(envInt("TICK_SECONDS", 60)) * time.Second
 	dailyTarget := envInt("DAILY_TARGET", 400) // swaps read per terminal per day
 	perTick := float64(dailyTarget) * tick.Seconds() / 86400
@@ -336,7 +346,7 @@ func main() {
 	// eth_getLogs) would otherwise hang every call for the whole timeout.
 	httpc := &http.Client{Timeout: 60 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
 	rps := envInt("RPC_RPS", 8)
-	rpc := &rpcClient{url: rpcURL, http: httpc, calls: cCalls.Inc, errors: cErrors.Inc, minGap: time.Second / time.Duration(max(rps, 1))}
+	rpc := &rpcClient{url: rpcURL, urls: rpcURLs, http: httpc, calls: cCalls.Inc, errors: cErrors.Inc, minGap: time.Second / time.Duration(max(rps, 1))}
 
 	applyRPCOverrides()
 	st := loadState(stateFile)
@@ -352,7 +362,7 @@ func main() {
 		// public wss://api.mainnet-beta.solana.com is free and keyless).
 		wsURL := os.Getenv("WS_URL")
 		if wsURL == "" {
-			wsURL = rpcURL
+			wsURL = "wss://api.mainnet-beta.solana.com"
 		}
 		fd = newFeed(wsURL)
 		go fd.run(context.Background())

harnesses/terminal-fill-quality/cmd/script/rpc.go

@@ -16,7 +16,8 @@ import (
 )
 
 type rpcClient struct {
-	url    string
+	url    string   // the primary endpoint
+	urls   []string // every endpoint in order: the next one is tried on a rate limit or a transport error
 	http   *http.Client
 	calls  func()
 	errors func()
@@ -59,8 +60,16 @@ func sleepCtx(ctx context.Context, d time.Duration) error {
 
 func (c *rpcClient) call(ctx context.Context, method string, params []any, out any) error {
 	body, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params})
+	urls := c.urls
+	if len(urls) == 0 {
+		urls = []string{c.url}
+	}
 	for attempt := 0; attempt < 4; attempt++ {
-		req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(body))
+		// The first attempt goes to the primary, each retry to the next
+		// endpoint (a rate limit or a dead node on one costs one hop, not
+		// the draw); the sleeps only apply when every endpoint was tried.
+		url := urls[attempt%len(urls)]
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
 		if err != nil {
 			return err
 		}
@@ -70,17 +79,21 @@ func (c *rpcClient) call(ctx context.Context, method string, params []any, out a
 		resp, err := c.http.Do(req)
 		if err != nil {
 			c.errors()
-			if err := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); err != nil {
-				return err
+			if attempt+1 >= len(urls) {
+				if err := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); err != nil {
+					return err
+				}
 			}
 			continue
 		}
 		data, _ := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
 		resp.Body.Close()
 		if resp.StatusCode == http.StatusTooManyRequests {
 			c.errors()
-			if err := sleepCtx(ctx, time.Duration(2*(attempt+1))*time.Second); err != nil {
-				return err
+			if attempt+1 >= len(urls) {
+				if err := sleepCtx(ctx, time.Duration(2*(attempt+1))*time.Second); err != nil {
+					return err
+				}
 			}
 			continue
 		}
@@ -98,8 +111,10 @@ func (c *rpcClient) call(ctx context.Context, method string, params []any, out a
 		if env.Error != nil {
 			if env.Error.Code == 429 || env.Error.Code == -32429 {
 				c.errors()
-				if err := sleepCtx(ctx, time.Duration(2*(attempt+1))*time.Second); err != nil {
-					return err
+				if attempt+1 >= len(urls) {
+					if err := sleepCtx(ctx, time.Duration(2*(attempt+1))*time.Second); err != nil {
+						return err
+					}
 				}
 				continue
 			}