Skip to content

MINOR: Preserve empty list offset buffer #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prashanthbdremio wants to merge 5 commits into
base: main
Choose a base branch
from
Open

MINOR: Preserve empty list offset buffer #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
858601a
DX-122953 Preserve empty list offset buffer
prashanthbdremio Jun 22, 2026
f5cfa00
DX-122953 Avoid NettyArrowBuf test import
prashanthbdremio Jun 22, 2026
10b259b
MINOR: Fix workflow trailing whitespace
prashanthbdremio Jun 22, 2026
e293989
MINOR: Reset list test helper capacity
prashanthbdremio Jun 23, 2026
91dafa2
Revert "MINOR: Reset list test helper capacity"
prashanthbdremio Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/jarbuild.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# under the License.

name: JarBuild
on:
on:
workflow_dispatch:
inputs:
arrow_branch:
Expand Down
16 changes: 14 additions & 2 deletions vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,15 +305,27 @@ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long

/** Set the reader and writer indexes for the inner buffers. */
private void setReaderAndWriterIndex() {
final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH;
validityBuffer.readerIndex(0);
offsetBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
offsetBuffer.writerIndex(0);
ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity);
} else {
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
}
// IPC serializers use readerIndex and writerIndex to determine readable bytes. Even when the
// list is empty, the Arrow layout requires the offset buffer to contain offset[0].
offsetBuffer.writerIndex(requiredOffsetBufferCapacity);
}

private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) {
if (offsetBuffer.capacity() >= requiredCapacity) {
return;
}
ArrowBuf oldOffsetBuffer = offsetBuffer;
offsetBuffer = allocateOffsetBuffer(requiredCapacity);
oldOffsetBuffer.getReferenceManager().release();
}

/**
Expand Down
16 changes: 14 additions & 2 deletions vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,15 +263,27 @@ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long

/** Set the reader and writer indexes for the inner buffers. */
private void setReaderAndWriterIndex() {
final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH;
validityBuffer.readerIndex(0);
offsetBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
offsetBuffer.writerIndex(0);
ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity);
} else {
validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH);
}
// IPC serializers use readerIndex and writerIndex to determine readable bytes. Even when the
// list is empty, the Arrow layout requires the offset buffer to contain offset[0].
offsetBuffer.writerIndex(requiredOffsetBufferCapacity);
}

private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) {
if (offsetBuffer.capacity() >= requiredCapacity) {
return;
}
ArrowBuf oldOffsetBuffer = offsetBuffer;
offsetBuffer = allocateOffsetBuffer(requiredCapacity);
oldOffsetBuffer.getReferenceManager().release();
}

/**
Expand Down
30 changes: 30 additions & 0 deletions vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,36 @@ public void testGetBufferSizeFor() {
}
}

@Test
public void testEmptyLargeListOffsetBuffer() {
try (LargeListVector list = LargeListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.allocateNew();
list.setValueCount(0);

assertEmptyLargeListOffsetBuffer(list);
}
}

@Test
public void testUnallocatedEmptyLargeListOffsetBuffer() {
try (LargeListVector list = LargeListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.setValueCount(0);

assertEmptyLargeListOffsetBuffer(list);
}
}

private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) {
List<ArrowBuf> buffers = list.getFieldBuffers();
ArrowBuf offsetBuffer = buffers.get(1);
assertEquals(LargeListVector.OFFSET_WIDTH, offsetBuffer.readableBytes());
assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH);
assertEquals(0L, offsetBuffer.getLong(0));
return offsetBuffer;
}

@Test
public void testIsEmpty() {
try (final LargeListVector vector = LargeListVector.empty("list", allocator)) {
Expand Down
30 changes: 30 additions & 0 deletions vector/src/test/java/org/apache/arrow/vector/TestListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,36 @@ public void testGetBufferSizeFor() {
}
}

@Test
public void testEmptyListOffsetBuffer() {
try (ListVector list = ListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.allocateNew();
list.setValueCount(0);

assertEmptyListOffsetBuffer(list);
}
}

@Test
public void testUnallocatedEmptyListOffsetBuffer() {
try (ListVector list = ListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.setValueCount(0);

assertEmptyListOffsetBuffer(list);
}
}

private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) {
List<ArrowBuf> buffers = list.getFieldBuffers();
ArrowBuf offsetBuffer = buffers.get(1);
assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, offsetBuffer.readableBytes());
assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH);
assertEquals(0, offsetBuffer.getInt(0));
return offsetBuffer;
}

@Test
public void testIsEmpty() {
try (final ListVector vector = ListVector.empty("list", allocator)) {
Expand Down
Loading