From ebe2e426ff5f2361fa86508d7ba027887c6eb3c8 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 20 Apr 2026 20:49:21 +0700 Subject: [PATCH] fix: set MaxBatchByteSize on SQS topic to prevent oversized batches gocloud.dev's SQS driver batches Send() calls internally via SendMessageBatch (up to 10 messages) but does not enforce SQS's 1 MB batch size limit. When log entries carry large payloads the batch can exceed 1 MB, causing SQS to reject the entire batch. This cascades into a permanent retry loop because the log entry never gets persisted and the retry scheduler cannot find a prior attempt. Set MaxBatchByteSize to 1 MB so gocloud splits oversized batches before they reach SQS. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/mqs/queue_awssqs.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/mqs/queue_awssqs.go b/internal/mqs/queue_awssqs.go index 79f59900..0e112302 100644 --- a/internal/mqs/queue_awssqs.go +++ b/internal/mqs/queue_awssqs.go @@ -13,8 +13,14 @@ import ( "github.com/aws/aws-sdk-go/aws" "gocloud.dev/pubsub" "gocloud.dev/pubsub/awssnssqs" + "gocloud.dev/pubsub/batcher" ) +// sqsMaxBatchBytes is the maximum total payload size for an SQS SendMessageBatch +// request. SQS rejects batches exceeding 256 KB per message and 1 MB per batch. +// Setting this on the gocloud batcher prevents oversized batches from being sent. +const sqsMaxBatchBytes = 1_048_576 // 1 MB + type AWSSQSConfig struct { Endpoint string // optional - dev-focused Region string @@ -56,7 +62,11 @@ func (q *AWSQueue) Init(ctx context.Context) (func(), error) { if err != nil { return nil, err } - q.topic = awssnssqs.OpenSQSTopicV2(ctx, q.sqsClient, q.sqsQueueURL, nil) + q.topic = awssnssqs.OpenSQSTopicV2(ctx, q.sqsClient, q.sqsQueueURL, &awssnssqs.TopicOptions{ + BatcherOptions: batcher.Options{ + MaxBatchByteSize: sqsMaxBatchBytes, + }, + }) return func() { q.topic.Shutdown(ctx) }, nil