Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit d2b16ea

Browse files
lqiu96lqiu96
committed
chore: Use INT64 for micros and String for picos timestamp field
1 parent a9fc238 commit d2b16ea

File tree

4 files changed

+156
-53
lines changed

4 files changed

+156
-53
lines changed

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,22 @@
3030
import java.util.ArrayList;
3131
import java.util.HashMap;
3232
import java.util.List;
33+
import java.util.Map;
3334

3435
/**
3536
* Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase
3637
* when constructing the protobuf descriptor. The mapping between field types and field modes are
3738
* shown in the ImmutableMaps below.
3839
*/
3940
public class BQTableSchemaToProtoDescriptor {
40-
private static ImmutableMap<TableFieldSchema.Mode, FieldDescriptorProto.Label>
41-
BQTableSchemaModeMap =
42-
ImmutableMap.of(
43-
TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL,
44-
TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED,
45-
TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED);
41+
private static Map<Mode, FieldDescriptorProto.Label> DEFAULT_BQ_TABLE_SCHEMA_MODE_MAP =
42+
ImmutableMap.of(
43+
TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL,
44+
TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED,
45+
TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED);
4646

47-
private static ImmutableMap<TableFieldSchema.Type, FieldDescriptorProto.Type>
48-
BQTableSchemaTypeMap =
47+
private static Map<TableFieldSchema.Type, FieldDescriptorProto.Type>
48+
DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP =
4949
new ImmutableMap.Builder<TableFieldSchema.Type, FieldDescriptorProto.Type>()
5050
.put(TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL)
5151
.put(TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES)
@@ -59,7 +59,7 @@ public class BQTableSchemaToProtoDescriptor {
5959
.put(TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
6060
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
6161
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
62-
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_STRING)
62+
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
6363
.put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING)
6464
.put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING)
6565
.put(TableFieldSchema.Type.RANGE, FieldDescriptorProto.Type.TYPE_MESSAGE)
@@ -142,11 +142,13 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
142142
.setType(BQTableField.getRangeElementType().getType())
143143
.setName("start")
144144
.setMode(Mode.NULLABLE)
145+
.setTimestampPrecision(BQTableField.getTimestampPrecision())
145146
.build(),
146147
TableFieldSchema.newBuilder()
147148
.setType(BQTableField.getRangeElementType().getType())
148149
.setName("end")
149150
.setMode(Mode.NULLABLE)
151+
.setTimestampPrecision(BQTableField.getTimestampPrecision())
150152
.build());
151153

152154
if (dependencyMap.containsKey(rangeFields)) {
@@ -189,7 +191,7 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
189191
* @param index Index for protobuf fields.
190192
* @param scope used to name descriptors
191193
*/
192-
private static FieldDescriptorProto convertBQTableFieldToProtoField(
194+
static FieldDescriptorProto convertBQTableFieldToProtoField(
193195
TableFieldSchema BQTableField, int index, String scope) {
194196
TableFieldSchema.Mode mode = BQTableField.getMode();
195197
String fieldName = BQTableField.getName().toLowerCase();
@@ -198,20 +200,32 @@ private static FieldDescriptorProto convertBQTableFieldToProtoField(
198200
FieldDescriptorProto.newBuilder()
199201
.setName(fieldName)
200202
.setNumber(index)
201-
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode));
203+
.setLabel((FieldDescriptorProto.Label) DEFAULT_BQ_TABLE_SCHEMA_MODE_MAP.get(mode));
202204

203205
switch (BQTableField.getType()) {
204206
case STRUCT:
205207
fieldDescriptor.setTypeName(scope);
206208
break;
207209
case RANGE:
208210
fieldDescriptor.setType(
209-
(FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()));
211+
(FieldDescriptorProto.Type)
212+
DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP.get(BQTableField.getType()));
210213
fieldDescriptor.setTypeName(scope);
211214
break;
215+
case TIMESTAMP:
216+
// Can map to either int64 or string based on the BQ Field's timestamp precision
217+
// Default: microsecond (6) maps to int64 and picosecond (12) maps to string
218+
if (BQTableField.getTimestampPrecision().getValue() == 12L) {
219+
fieldDescriptor.setType(
220+
(FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_STRING);
221+
} else {
222+
fieldDescriptor.setType((FieldDescriptorProto.Type) FieldDescriptorProto.Type.TYPE_INT64);
223+
}
224+
break;
212225
default:
213226
fieldDescriptor.setType(
214-
(FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()));
227+
(FieldDescriptorProto.Type)
228+
DEFAULT_BQ_TABLE_SCHEMA_TYPE_MAP.get(BQTableField.getType()));
215229
break;
216230
}
217231

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,26 @@ private void fillField(
633633
protoMsg.setField(fieldDescriptor, val);
634634
return;
635635
}
636+
} else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
637+
if (val instanceof String) {
638+
Double parsed = Doubles.tryParse((String) val);
639+
if (parsed != null) {
640+
protoMsg.setField(fieldDescriptor, parsed.longValue());
641+
return;
642+
}
643+
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
644+
protoMsg.setField(
645+
fieldDescriptor,
646+
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
647+
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
648+
return;
649+
} else if (val instanceof Long) {
650+
protoMsg.setField(fieldDescriptor, val);
651+
return;
652+
} else if (val instanceof Integer) {
653+
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val));
654+
return;
655+
}
636656
}
637657
}
638658
if (val instanceof Integer) {
@@ -682,14 +702,14 @@ private void fillField(
682702
// BQ Table Timestamp Fields are transmitted as a String. Check this first
683703
// as numeric values may actually represent a timestamp (micros from epoch).
684704
// Supported types: https://docs.cloud.google.com/bigquery/docs/supported-data-types
685-
if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
705+
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
686706
if (val instanceof String) {
687707
String value = (String) val;
688708
if (isValidTimestamp(value)) {
689709
protoMsg.setField(fieldDescriptor, value);
690710
}
691711
} else if (val instanceof Short || val instanceof Integer || val instanceof Long) {
692-
protoMsg.setField(fieldDescriptor, fromEpochMicros((Long) val).toString());
712+
protoMsg.setField(fieldDescriptor, ((Long) val).toString());
693713
} else if (val instanceof Timestamp) {
694714
Timestamp timestamp = (Timestamp) val;
695715
protoMsg.setField(
@@ -974,14 +994,14 @@ private void fillRepeatedField(
974994
// BQ Table Timestamp Fields are transmitted as a String. Check this first
975995
// as numeric values may actually represent a timestamp (micros from epoch).
976996
// Supported types: https://docs.cloud.google.com/bigquery/docs/supported-data-types
977-
if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
997+
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
978998
if (val instanceof String) {
979999
String value = (String) val;
9801000
if (isValidTimestamp(value)) {
9811001
protoMsg.addRepeatedField(fieldDescriptor, value);
9821002
}
9831003
} else if (val instanceof Short || val instanceof Integer || val instanceof Long) {
984-
protoMsg.addRepeatedField(fieldDescriptor, fromEpochMicros((Long) val).toString());
1004+
protoMsg.setField(fieldDescriptor, ((Long) val).toString());
9851005
} else if (val instanceof Timestamp) {
9861006
Timestamp timestamp = (Timestamp) val;
9871007
protoMsg.addRepeatedField(

0 commit comments

Comments
 (0)