Skip to content

Commit 6d12534

Browse files
cxzl25dongjoon-hyun
authored andcommitted
ORC-2177: Fix array conversion with empty first batch
### What changes were proposed in this pull request? Add a `batchSize > 0` guard in `ConvertTreeReader.convertVector()` before accessing index 0. ### Why are the changes needed? When reading an ORC file with schema evolution converting `array<int>` to `array<string>`, an `ArrayIndexOutOfBoundsException` is thrown if the first batch processed by the element reader has `childCount=0` (i.e., all rows in the batch contain empty arrays). ```java java.lang.ArrayIndexOutOfBoundsException: Index 0 out of bounds for length 0 at org.apache.orc.impl.ConvertTreeReaderFactory$StringGroupFromAnyIntegerTreeReader.setConvertVectorElement(ConvertTreeReaderFactory.java:1094) at org.apache.orc.impl.ConvertTreeReaderFactory$ConvertTreeReader.convertVector(ConvertTreeReaderFactory.java:305) at org.apache.orc.impl.ConvertTreeReaderFactory$StringGroupFromAnyIntegerTreeReader.nextVector(ConvertTreeReaderFactory.java:1115) at org.apache.orc.impl.TreeReaderFactory$ListTreeReader.nextVector(TreeReaderFactory.java:2892) at org.apache.orc.impl.reader.tree.StructBatchReader.readBatchColumn(StructBatchReader.java:66) at org.apache.orc.impl.reader.tree.StructBatchReader.nextBatchForLevel(StructBatchReader.java:101) at org.apache.orc.impl.reader.tree.StructBatchReader.nextBatch(StructBatchReader.java:78) at org.apache.orc.impl.RecordReaderImpl.nextBatch(RecordReaderImpl.java:1444) ``` ### How was this patch tested? Added `testIntArrayToStringArrayFirstBatchAllEmpty` in `TestConvertTreeReaderFactory` ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Closes #2638 from cxzl25/ORC-2177. Authored-by: sychen <sychen@ctrip.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
1 parent 325efd3 commit 6d12534

2 files changed

Lines changed: 41 additions & 5 deletions

File tree

java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -301,11 +301,13 @@ public void convertVector(ColumnVector fromColVector,
301301
resultColVector.reset();
302302
if (fromColVector.isRepeating) {
303303
resultColVector.isRepeating = true;
304-
if (fromColVector.noNulls || !fromColVector.isNull[0]) {
305-
setConvertVectorElement(0);
306-
} else {
307-
resultColVector.noNulls = false;
308-
resultColVector.isNull[0] = true;
304+
if (batchSize > 0) {
305+
if (fromColVector.noNulls || !fromColVector.isNull[0]) {
306+
setConvertVectorElement(0);
307+
} else {
308+
resultColVector.noNulls = false;
309+
resultColVector.isNull[0] = true;
310+
}
309311
}
310312
} else if (fromColVector.noNulls) {
311313
for (int i = 0; i < batchSize; i++) {

java/core/src/test/org/apache/orc/impl/TestConvertTreeReaderFactory.java

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,4 +746,38 @@ private void testDecimalConvertToDecimalInNullStripe() throws Exception {
746746
readDecimalInNullStripe("decimal(18,2)", DecimalColumnVector.class,
747747
new String[]{"null", "1024", "1"});
748748
}
749+
750+
@Test
751+
public void testIntArrayToStringArrayFirstBatchAllEmpty() throws Exception {
752+
TypeDescription fileSchema = TypeDescription.fromString("struct<col1:array<int>>");
753+
TypeDescription readerSchema = TypeDescription.fromString("struct<col1:array<string>>");
754+
755+
try (Writer w = OrcFile.createWriter(testFilePath,
756+
OrcFile.writerOptions(conf).setSchema(fileSchema))) {
757+
VectorizedRowBatch b = fileSchema.createRowBatch(3);
758+
ListColumnVector lc = (ListColumnVector) b.cols[0];
759+
for (int i = 0; i < 3; i++) {
760+
lc.offsets[i] = 0;
761+
lc.lengths[i] = 0;
762+
}
763+
lc.childCount = 0;
764+
b.size = 3;
765+
w.addRowBatch(b);
766+
}
767+
768+
try (Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
769+
RecordReader rows = reader.rows(reader.options().schema(readerSchema))) {
770+
VectorizedRowBatch rb = readerSchema.createRowBatch(3);
771+
assertTrue(rows.nextBatch(rb));
772+
ListColumnVector r = (ListColumnVector) rb.cols[0];
773+
// Cast verifies schema evolution took effect (would be LongColumnVector without evolution)
774+
BytesColumnVector child = (BytesColumnVector) r.child;
775+
assertEquals(0, r.childCount);
776+
for (int i = 0; i < 3; i++) {
777+
assertEquals(0, r.lengths[i], "row " + i + " should be empty array");
778+
}
779+
} finally {
780+
fs.delete(testFilePath, false);
781+
}
782+
}
749783
}

0 commit comments

Comments
 (0)