Skip to content

Commit 0ae5e16

Browse files
sambsnydsambsnyd
authored
Fix failing to read rows with cells larger than 4096 characters. (#7293)
Fix getRows() failing to look up the correct row type on javascript repositories
1 parent dcfe8ed commit 0ae5e16

File tree

6 files changed

+84
-37
lines changed

6 files changed

+84
-37
lines changed

‎rewrite-core/src/main/java/org/openrewrite/CsvDataTableStore.java‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
import java.io.*;
3030
import java.lang.reflect.Field;
31+
import java.lang.reflect.ParameterizedType;
3132
import java.nio.charset.StandardCharsets;
3233
import java.nio.file.Files;
3334
import java.nio.file.Path;
@@ -90,6 +91,10 @@ public class CsvDataTableStore implements DataTableStore, AutoCloseable {
9091
private final String fileExtension;
9192
private final Map<String, String> prefixColumns;
9293
private final Map<String, String> suffixColumns;
94+
private static final ObjectMapper ROW_MAPPER = new ObjectMapper()
95+
.registerModule(new ParameterNamesModule())
96+
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
97+
9398
private final ConcurrentHashMap<String, BucketWriter> writers = new ConcurrentHashMap<>();
9499
private final ConcurrentHashMap<String, RowMetadata> rowMetadata = new ConcurrentHashMap<>();
95100
private final ConcurrentHashMap<String, DataTable<?>> knownTables = new ConcurrentHashMap<>();
@@ -180,15 +185,30 @@ private static InputStream defaultInputStream(Path path) {
180185
@Override
181186
public <Row> void insertRow(DataTable<Row> dataTable, ExecutionContext ctx, Row row) {
182187
String metaKey = metaKey(dataTable.getName(), dataTable.getGroup());
183-
rowMetadata.computeIfAbsent(metaKey, k -> RowMetadata.from(dataTable));
188+
rowMetadata.computeIfAbsent(metaKey, k -> RowMetadata.of(dataTable.getType()));
184189
knownTables.putIfAbsent(fileKey(dataTable), dataTable);
185190
String fileKey = fileKey(dataTable);
186191
BucketWriter writer = writers.computeIfAbsent(fileKey, k -> createBucketWriter(dataTable));
187192
writer.writeRow(row);
188193
}
189194

195+
@Deprecated
190196
@Override
191197
public Stream<?> getRows(String dataTableName, @Nullable String group) {
198+
RowMetadata meta = rowMetadata.get(metaKey(dataTableName, group));
199+
return readRows(dataTableName, group, meta);
200+
}
201+
202+
@SuppressWarnings("unchecked")
203+
@Override
204+
public <Row> Stream<Row> getRows(Class<? extends DataTable<Row>> dataTableClass, @Nullable String group) {
205+
Class<Row> rowType = (Class<Row>) ((ParameterizedType) dataTableClass.getGenericSuperclass())
206+
.getActualTypeArguments()[0];
207+
return readRows(dataTableClass.getName(), group, RowMetadata.of(rowType));
208+
}
209+
210+
@SuppressWarnings("unchecked")
211+
private <T> Stream<T> readRows(String dataTableName, @Nullable String group, @Nullable RowMetadata meta) {
192212
// Close (not just flush) matching writers so that compression trailers
193213
// (e.g., GZIP footer) are written, making the files fully readable.
194214
// Removed writers will be lazily re-created in append mode on the next insertRow().
@@ -203,8 +223,6 @@ public Stream<?> getRows(String dataTableName, @Nullable String group) {
203223
}
204224
}
205225

206-
RowMetadata meta = rowMetadata.get(metaKey(dataTableName, group));
207-
208226
List<Object> allRows = new ArrayList<>();
209227
//noinspection DataFlowIssue
210228
File[] files = outputDir.toFile().listFiles((dir, name) -> name.endsWith(fileExtension));
@@ -241,6 +259,7 @@ public Stream<?> getRows(String dataTableName, @Nullable String group) {
241259

242260
try (InputStream is = inputStreamFactory.apply(file.toPath())) {
243261
CsvParserSettings settings = new CsvParserSettings();
262+
settings.setMaxCharsPerColumn(-1);
244263
settings.setHeaderExtractionEnabled(true);
245264
settings.getFormat().setComment('#');
246265
CsvParser parser = new CsvParser(settings);
@@ -265,7 +284,7 @@ public Stream<?> getRows(String dataTableName, @Nullable String group) {
265284
}
266285
}
267286

268-
return allRows.stream();
287+
return (Stream<T>) allRows.stream();
269288
}
270289

271290
@Override
@@ -385,44 +404,35 @@ private static String metaKey(String dataTableName, @Nullable String group) {
385404
}
386405

387406
/**
388-
* Holds the row class and its @Column field names so that
389-
* String[] rows read from CSV can be deserialized back to typed objects
390-
* via Jackson's {@link ObjectMapper#convertValue}.
407+
* Caches the {@link Column @Column} field names for a row class so they
408+
* are only computed once, and converts CSV {@code String[]} rows back to
409+
* typed objects via Jackson.
391410
*/
392411
private static class RowMetadata {
393-
private static final ObjectMapper MAPPER = new ObjectMapper()
394-
.registerModule(new ParameterNamesModule())
395-
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
396-
397-
final String rowClassName;
412+
final Class<?> rowClass;
398413
final List<String> fieldNames;
399414

400-
RowMetadata(String rowClassName, List<String> fieldNames) {
401-
this.rowClassName = rowClassName;
415+
private RowMetadata(Class<?> rowClass, List<String> fieldNames) {
416+
this.rowClass = rowClass;
402417
this.fieldNames = fieldNames;
403418
}
404419

405-
static RowMetadata from(DataTable<?> dataTable) {
406-
Class<?> rowClass = dataTable.getType();
420+
static RowMetadata of(Class<?> rowClass) {
407421
List<String> names = new ArrayList<>();
408422
for (Field f : rowClass.getDeclaredFields()) {
409423
if (f.isAnnotationPresent(Column.class)) {
410424
names.add(f.getName());
411425
}
412426
}
413-
return new RowMetadata(rowClass.getName(), names);
427+
return new RowMetadata(rowClass, names);
414428
}
415429

416430
Object toRow(String[] values) {
417431
Map<String, String> map = new LinkedHashMap<>();
418432
for (int i = 0; i < fieldNames.size(); i++) {
419433
map.put(fieldNames.get(i), i < values.length ? values[i] : "");
420434
}
421-
try {
422-
return MAPPER.convertValue(map, Class.forName(rowClassName));
423-
} catch (ClassNotFoundException e) {
424-
throw new IllegalStateException("Row class not found: " + rowClassName, e);
425-
}
435+
return ROW_MAPPER.convertValue(map, rowClass);
426436
}
427437
}
428438

@@ -486,7 +496,7 @@ public static String sanitize(String value) {
486496
prefix = prefix.substring(0, lastDash);
487497
}
488498
}
489-
String hash = sha256Prefix(value, 4);
499+
String hash = sha256Prefix(value);
490500
return prefix + "-" + hash;
491501
}
492502

@@ -528,18 +538,18 @@ public static String sanitize(String value) {
528538
return new DataTableDescriptor(name, name, instanceName, "", group, Collections.emptyList());
529539
}
530540

531-
private static String sha256Prefix(String input, int hexChars) {
541+
private static String sha256Prefix(String input) {
532542
try {
533543
MessageDigest digest = MessageDigest.getInstance("SHA-256");
534544
byte[] hash = digest.digest(input.getBytes(StandardCharsets.UTF_8));
535545
StringBuilder hex = new StringBuilder();
536546
for (byte b : hash) {
537547
hex.append(String.format("%02x", b));
538-
if (hex.length() >= hexChars) {
548+
if (hex.length() >= 4) {
539549
break;
540550
}
541551
}
542-
return hex.substring(0, Math.min(hexChars, hex.length()));
552+
return hex.substring(0, Math.min(4, hex.length()));
543553
} catch (NoSuchAlgorithmException e) {
544554
throw new IllegalStateException(e);
545555
}

‎rewrite-core/src/main/java/org/openrewrite/DataTableStore.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,36 @@ static DataTableStore noop() {
6060
* @param dataTableName the fully qualified class name of the data table
6161
* @param group the group identifying the bucket, or null for ungrouped
6262
* @return a stream of rows, or an empty stream if no rows exist
63+
* @deprecated Use {@link #getRows(Class)} or {@link #getRows(Class, String)} for type-safe deserialization.
6364
*/
65+
@Deprecated
6466
Stream<?> getRows(String dataTableName, @Nullable String group);
6567

68+
/**
69+
* Stream typed rows for a specific data table class and group.
70+
* The row type is inferred from the data table's generic parameter.
71+
*
72+
* @param dataTableClass the data table class (e.g., {@code ServiceEndpoints.class})
73+
* @param group the group identifying the bucket, or null for ungrouped
74+
* @param <Row> the row type
75+
* @return a stream of typed rows, or an empty stream if no rows exist
76+
*/
77+
@SuppressWarnings("unchecked")
78+
default <Row> Stream<Row> getRows(Class<? extends DataTable<Row>> dataTableClass, @Nullable String group) {
79+
return (Stream<Row>) getRows(dataTableClass.getName(), group);
80+
}
81+
82+
/**
83+
* Stream typed rows for a specific data table class (ungrouped).
84+
*
85+
* @param dataTableClass the data table class (e.g., {@code ServiceEndpoints.class})
86+
* @param <Row> the row type
87+
* @return a stream of typed rows, or an empty stream if no rows exist
88+
*/
89+
default <Row> Stream<Row> getRows(Class<? extends DataTable<Row>> dataTableClass) {
90+
return getRows(dataTableClass, null);
91+
}
92+
6693
/**
6794
* Get the set of {@link DataTable} instances that have received rows.
6895
*

‎rewrite-core/src/main/java/org/openrewrite/RecipeRun.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,30 @@ public class RecipeRun {
4444
return null;
4545
}
4646

47+
/**
48+
* @deprecated Use {@link #getDataTableRows(Class)} for type-safe deserialization.
49+
*/
50+
@Deprecated
4751
public <E> List<E> getDataTableRows(String name) {
4852
return getDataTableRows(name, null);
4953
}
5054

55+
/**
56+
* @deprecated Use {@link #getDataTableRows(Class, String)} for type-safe deserialization.
57+
*/
5158
@SuppressWarnings("unchecked")
59+
@Deprecated
5260
public <E> List<E> getDataTableRows(String name, @Nullable String group) {
5361
return (List<E>) dataTableStore.getRows(name, group)
5462
.collect(Collectors.toList());
5563
}
64+
65+
public <E> List<E> getDataTableRows(Class<? extends DataTable<E>> dataTableClass) {
66+
return getDataTableRows(dataTableClass, null);
67+
}
68+
69+
public <E> List<E> getDataTableRows(Class<? extends DataTable<E>> dataTableClass, @Nullable String group) {
70+
return dataTableStore.getRows(dataTableClass, group)
71+
.collect(Collectors.toList());
72+
}
5673
}

‎rewrite-core/src/test/java/org/openrewrite/DataTableStoreTest.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -486,14 +486,8 @@ public List<String> getInitialValue(ExecutionContext ctx) {
486486
// On cycle 2+, the store already contains rows written in cycle 1
487487
List<String> readBack = new ArrayList<>();
488488
DataTableStore store = DataTableExecutionContextView.view(ctx).getDataTableStore();
489-
try (Stream<?> rows = store.getRows(table.getName(), null)) {
490-
rows.forEach(row -> {
491-
if (row instanceof TestTable.Row) {
492-
readBack.add(((TestTable.Row) row).getName());
493-
} else {
494-
readBack.add(((String[]) row)[0]);
495-
}
496-
});
489+
try (Stream<TestTable.Row> rows = store.getRows(TestTable.class)) {
490+
rows.forEach(row -> readBack.add(row.getName()));
497491
}
498492
return readBack;
499493
}

‎rewrite-java-test/src/test/java/org/openrewrite/java/search/FindFieldsOfTypeTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ void verifyDataTable() {
136136
rewriteRun(
137137
spec -> spec.recipe(new FindFieldsOfType("java.util.List", true))
138138
.afterRecipe(recipeRun -> {
139-
List<FieldsOfTypeUses.Row> fields = recipeRun.getDataTableRows(FieldsOfTypeUses.class.getName());
139+
List<FieldsOfTypeUses.Row> fields = recipeRun.getDataTableRows(FieldsOfTypeUses.class);
140140
assertThat(fields).containsExactlyInAnyOrderElementsOf(expectedFields);
141141
}),
142142
java(

‎rewrite-test/src/main/java/org/openrewrite/test/RecipeSpec.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,7 @@ public <E, V> RecipeSpec dataTableAsCsv(String name, String expect) {
250250
}
251251
}
252252
assertThat(dataTable).isNotNull();
253-
@SuppressWarnings("unchecked")
254-
List<E> rows = (List<E>) store.getRows(dataTable.getName(), dataTable.getGroup())
253+
List<?> rows = store.getRows(dataTable.getName(), dataTable.getGroup())
255254
.collect(java.util.stream.Collectors.toList());
256255
StringWriter writer = new StringWriter();
257256
CsvMapper mapper = CsvMapper.builder()

0 commit comments

Comments
 (0)