-
Notifications
You must be signed in to change notification settings - Fork 25
fix: BatchWriter: Ensure flushed batch has at least one record #2489
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -161,8 +161,9 @@ func (w *BatchWriter) worker(ctx context.Context, tableName string, ch <-chan *m | |
| limit.AddSlice(add) | ||
| } | ||
| if len(toFlush) > 0 || rest != nil || limit.ReachedLimit() { | ||
| // flush current batch | ||
| send() | ||
| if limit.Rows() > 0 { | ||
| send() | ||
| } | ||
| ticker.Reset(w.batchTimeout) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this go inside the if as well?
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Or have |
||
| } | ||
| for _, sliceToFlush := range toFlush { | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -233,6 +233,57 @@ func TestBatchUpserts(t *testing.T) { | |||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| // noEmptyBatchClient wraps testBatchClient and fails the test if WriteTableBatch | ||||||||
| // is called with an empty messages slice. | ||||||||
| type noEmptyBatchClient struct { | ||||||||
| testBatchClient | ||||||||
| t *testing.T | ||||||||
| } | ||||||||
|
|
||||||||
| func (c *noEmptyBatchClient) WriteTableBatch(ctx context.Context, name string, messages message.WriteInserts) error { | ||||||||
| if len(messages) == 0 { | ||||||||
| // Use t.Error (not t.Fatal) because this may be called from a worker goroutine. | ||||||||
| c.t.Error("WriteTableBatch called with empty messages slice") | ||||||||
| return nil | ||||||||
| } | ||||||||
| return c.testBatchClient.WriteTableBatch(ctx, name, messages) | ||||||||
| } | ||||||||
|
|
||||||||
| // TestBatchNoEmptyFlush is a regression test ensuring WriteTableBatch is never called with | ||||||||
| // an empty messages slice. This can happen when batchSizeBytes is so small that no row fits | ||||||||
| // in the initial batch (SliceRecord returns add==nil while toFlush is non-empty), which | ||||||||
| // previously caused send() to call WriteTableBatch with an empty resources slice. | ||||||||
| func TestBatchNoEmptyFlush(t *testing.T) { | ||||||||
| ctx := context.Background() | ||||||||
|
|
||||||||
| testClient := &noEmptyBatchClient{t: t} | ||||||||
| // batchSizeBytes=1 ensures that no single row fits in the initial batch: | ||||||||
| // SliceRecord returns add==nil, toFlush=[one record per row], rest=nil. | ||||||||
|
||||||||
| // SliceRecord returns add==nil, toFlush=[one record per row], rest=nil. | |
| // SliceRecord returns add==nil, toFlush=[one record per row], and a non-nil remaining | |
| // record for the unsliced original batch. |
Uh oh!
There was an error while loading. Please reload this page.