From c0b874aa0150e63908450b13d019864b8cbfbfe3 Mon Sep 17 00:00:00 2001 From: Liam <44932470+whuffman36@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:25:38 -0800 Subject: [PATCH] feat: Enable Lossless Timestamps in BQ java client lib (#3589) * feat: Enable Lossless Timestamps in BQ java client lib * Fix Formatting. * Fix tests for FieldValue and FieldValueList. * Add more robust testing to IT test, minor formatting fixes. --- .../google/cloud/bigquery/BigQueryImpl.java | 18 ++++--- .../cloud/bigquery/BigQueryOptions.java | 18 ++++++- .../com/google/cloud/bigquery/FieldValue.java | 51 ++++++++++++++----- .../google/cloud/bigquery/FieldValueList.java | 8 ++- .../cloud/bigquery/QueryRequestInfo.java | 12 ++++- .../cloud/bigquery/FieldValueListTest.java | 26 ++++++++++ .../google/cloud/bigquery/FieldValueTest.java | 19 +++++++ .../cloud/bigquery/QueryRequestInfoTest.java | 23 +++++++-- .../cloud/bigquery/it/ITBigQueryTest.java | 36 +++++++++++++ 9 files changed, 183 insertions(+), 28 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 576083215..770345000 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -1206,7 +1206,7 @@ public TableDataList call() { new PageImpl<>( new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap), cursor, - transformTableData(result.getRows(), schema)), + transformTableData(result.getRows(), schema, serviceOptions.getUseInt64Timestamps())), result.getTotalRows()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); @@ -1214,7 +1214,7 @@ public TableDataList call() { } private static Iterable transformTableData( - Iterable tableDataPb, final Schema schema) { + Iterable tableDataPb, final Schema schema, boolean useInt64Timestamps) { return ImmutableList.copyOf( Iterables.transform( tableDataPb != null ? tableDataPb : ImmutableList.of(), @@ -1223,7 +1223,7 @@ private static Iterable transformTableData( @Override public FieldValueList apply(TableRow rowPb) { - return FieldValueList.fromPb(rowPb.getF(), fields); + return FieldValueList.fromPb(rowPb.getF(), fields, useInt64Timestamps); } })); } @@ -1347,7 +1347,8 @@ public TableResult query(QueryJobConfiguration configuration, JobOption... optio // If all parameters passed in configuration are supported by the query() method on the backend, // put on fast path - QueryRequestInfo requestInfo = new QueryRequestInfo(configuration); + QueryRequestInfo requestInfo = + new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); if (requestInfo.isFastQuerySupported(null)) { String projectId = getOptions().getProjectId(); QueryRequest content = requestInfo.toPb(); @@ -1420,7 +1421,8 @@ public com.google.api.services.bigquery.model.QueryResponse call() { // fetch next pages of results new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)), cursor, - transformTableData(results.getRows(), schema))) + transformTableData( + results.getRows(), schema, getOptions().getUseInt64Timestamps()))) .setJobId(jobId) .setQueryId(results.getQueryId()) .build(); @@ -1433,7 +1435,8 @@ public com.google.api.services.bigquery.model.QueryResponse call() { new PageImpl<>( new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)), null, - transformTableData(results.getRows(), schema))) + transformTableData( + results.getRows(), schema, getOptions().getUseInt64Timestamps()))) // Return the JobID of the successful job .setJobId( results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null) @@ -1448,7 +1451,8 @@ public TableResult query(QueryJobConfiguration configuration, JobId jobId, JobOp // If all parameters passed in configuration are supported by the query() method on the backend, // put on fast path - QueryRequestInfo requestInfo = new QueryRequestInfo(configuration); + QueryRequestInfo requestInfo = + new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); if (requestInfo.isFastQuerySupported(jobId)) { // Be careful when setting the projectID in JobId, if a projectID is specified in the JobId, // the job created by the query method will use that project. This may cause the query to diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java index e53439f02..465cc8305 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java @@ -34,10 +34,11 @@ public class BigQueryOptions extends ServiceOptions { private static final int DEFAULT_READ_API_TIME_OUT = 60000; private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/bigquery"; private static final Set SCOPES = ImmutableSet.of(BIGQUERY_SCOPE); - private static final long serialVersionUID = -2437598817433266049L; + private static final long serialVersionUID = -2437598817433266048L; private final String location; // set the option ThrowNotFound when you want to throw the exception when the value not found private boolean setThrowNotFound; + private boolean useInt64Timestamps; private String queryPreviewEnabled = System.getenv("QUERY_PREVIEW_ENABLED"); public static class DefaultBigQueryFactory implements BigQueryFactory { @@ -63,6 +64,7 @@ public ServiceRpc create(BigQueryOptions options) { public static class Builder extends ServiceOptions.Builder { private String location; + private boolean useInt64Timestamps; private Builder() {} @@ -84,6 +86,11 @@ public Builder setLocation(String location) { return this; } + public Builder setUseInt64Timestamps(boolean useInt64Timestamps) { + this.useInt64Timestamps = useInt64Timestamps; + return this; + } + @Override public BigQueryOptions build() { return new BigQueryOptions(this); @@ -93,6 +100,7 @@ public BigQueryOptions build() { private BigQueryOptions(Builder builder) { super(BigQueryFactory.class, BigQueryRpcFactory.class, builder, new BigQueryDefaults()); this.location = builder.location; + this.useInt64Timestamps = builder.useInt64Timestamps; } private static class BigQueryDefaults implements ServiceDefaults { @@ -140,6 +148,10 @@ public void setThrowNotFound(boolean setThrowNotFound) { this.setThrowNotFound = setThrowNotFound; } + public void setUseInt64Timestamps(boolean useInt64Timestamps) { + this.useInt64Timestamps = useInt64Timestamps; + } + @VisibleForTesting public void setQueryPreviewEnabled(String queryPreviewEnabled) { this.queryPreviewEnabled = queryPreviewEnabled; @@ -149,6 +161,10 @@ public boolean getThrowNotFound() { return setThrowNotFound; } + public boolean getUseInt64Timestamps() { + return useInt64Timestamps; + } + @SuppressWarnings("unchecked") @Override public Builder toBuilder() { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java index c5a8fab07..d11df4b95 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValue.java @@ -26,6 +26,7 @@ import com.google.common.io.BaseEncoding; import java.io.Serializable; import java.math.BigDecimal; +import java.math.BigInteger; import java.math.RoundingMode; import java.time.Duration; import java.time.Instant; @@ -46,10 +47,11 @@ public class FieldValue implements Serializable { private static final int MICROSECONDS = 1000000; - private static final long serialVersionUID = 469098630191710061L; + private static final long serialVersionUID = 469098630191710062L; private final Attribute attribute; private final Object value; + private final Boolean useInt64Timestamps; /** The field value's attribute, giving information on the field's content type. */ public enum Attribute { @@ -74,8 +76,13 @@ public enum Attribute { } private FieldValue(Attribute attribute, Object value) { + this(attribute, value, false); + } + + private FieldValue(Attribute attribute, Object value, Boolean useInt64Timestamps) { this.attribute = checkNotNull(attribute); this.value = value; + this.useInt64Timestamps = useInt64Timestamps; } /** @@ -107,6 +114,10 @@ public Object getValue() { return value; } + public Boolean getUseInt64Timestamps() { + return useInt64Timestamps; + } + /** * Returns this field's value as a {@link String}. This method should only be used if the * corresponding field has primitive type ({@link LegacySQLTypeName#BYTES}, {@link @@ -207,6 +218,9 @@ public boolean getBooleanValue() { */ @SuppressWarnings("unchecked") public long getTimestampValue() { + if (useInt64Timestamps) { + return new BigInteger(getStringValue()).longValue(); + } // timestamps are encoded in the format 1408452095.22 where the integer part is seconds since // epoch (e.g. 1408452095.22 == 2014-08-19 07:41:35.220 -05:00) BigDecimal secondsWithMicro = new BigDecimal(getStringValue()); @@ -317,12 +331,13 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("attribute", attribute) .add("value", value) + .add("useInt64Timestamps", useInt64Timestamps) .toString(); } @Override public final int hashCode() { - return Objects.hash(attribute, value); + return Objects.hash(attribute, value, useInt64Timestamps); } @Override @@ -334,7 +349,9 @@ public final boolean equals(Object obj) { return false; } FieldValue other = (FieldValue) obj; - return attribute == other.attribute && Objects.equals(value, other.value); + return attribute == other.attribute + && Objects.equals(value, other.value) + && Objects.equals(useInt64Timestamps, other.useInt64Timestamps); } /** @@ -353,29 +370,38 @@ public final boolean equals(Object obj) { */ @BetaApi public static FieldValue of(Attribute attribute, Object value) { - return new FieldValue(attribute, value); + return of(attribute, value, false); + } + + @BetaApi + public static FieldValue of(Attribute attribute, Object value, Boolean useInt64Timestamps) { + return new FieldValue(attribute, value, useInt64Timestamps); } static FieldValue fromPb(Object cellPb) { - return fromPb(cellPb, null); + return fromPb(cellPb, null, false); } @SuppressWarnings("unchecked") - static FieldValue fromPb(Object cellPb, Field recordSchema) { + static FieldValue fromPb(Object cellPb, Field recordSchema, Boolean useInt64Timestamps) { if (Data.isNull(cellPb)) { - return FieldValue.of(Attribute.PRIMITIVE, null); + return FieldValue.of(Attribute.PRIMITIVE, null, useInt64Timestamps); } if (cellPb instanceof String) { if ((recordSchema != null) && (recordSchema.getType() == LegacySQLTypeName.RANGE) && (recordSchema.getRangeElementType() != null)) { return FieldValue.of( - Attribute.RANGE, Range.of((String) cellPb, recordSchema.getRangeElementType())); + Attribute.RANGE, + Range.of((String) cellPb, recordSchema.getRangeElementType()), + useInt64Timestamps); } - return FieldValue.of(Attribute.PRIMITIVE, cellPb); + return FieldValue.of(Attribute.PRIMITIVE, cellPb, useInt64Timestamps); } if (cellPb instanceof List) { - return FieldValue.of(Attribute.REPEATED, FieldValueList.fromPb((List) cellPb, null)); + return FieldValue.of( + Attribute.REPEATED, + FieldValueList.fromPb((List) cellPb, null, useInt64Timestamps)); } if (cellPb instanceof Map) { Map cellMapPb = (Map) cellPb; @@ -383,12 +409,13 @@ static FieldValue fromPb(Object cellPb, Field recordSchema) { FieldList subFieldsSchema = recordSchema != null ? recordSchema.getSubFields() : null; return FieldValue.of( Attribute.RECORD, - FieldValueList.fromPb((List) cellMapPb.get("f"), subFieldsSchema)); + FieldValueList.fromPb( + (List) cellMapPb.get("f"), subFieldsSchema, useInt64Timestamps)); } // This should never be the case when we are processing a first level table field (i.e. a // row's field, not a record sub-field) if (cellMapPb.containsKey("v")) { - return FieldValue.fromPb(cellMapPb.get("v"), recordSchema); + return FieldValue.fromPb(cellMapPb.get("v"), recordSchema, useInt64Timestamps); } } throw new IllegalArgumentException("Unexpected table cell format"); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValueList.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValueList.java index 5035bb164..18d2155a5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValueList.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FieldValueList.java @@ -112,6 +112,10 @@ FieldValueList withSchema(FieldList schema) { } static FieldValueList fromPb(List rowPb, FieldList schema) { + return fromPb(rowPb, schema, false); + } + + static FieldValueList fromPb(List rowPb, FieldList schema, Boolean useInt64Timestamps) { List row = new ArrayList<>(rowPb.size()); if (schema != null) { if (schema.size() != rowPb.size()) { @@ -120,11 +124,11 @@ static FieldValueList fromPb(List rowPb, FieldList schema) { Iterator schemaIter = schema.iterator(); Iterator rowPbIter = rowPb.iterator(); while (rowPbIter.hasNext() && schemaIter.hasNext()) { - row.add(FieldValue.fromPb(rowPbIter.next(), schemaIter.next())); + row.add(FieldValue.fromPb(rowPbIter.next(), schemaIter.next(), useInt64Timestamps)); } } else { for (Object cellPb : rowPb) { - row.add(FieldValue.fromPb(cellPb, null)); + row.add(FieldValue.fromPb(cellPb, null, useInt64Timestamps)); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java index 00a11f723..1a56872a1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.api.services.bigquery.model.DataFormatOptions; import com.google.api.services.bigquery.model.QueryParameter; import com.google.api.services.bigquery.model.QueryRequest; import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode; @@ -42,8 +43,9 @@ final class QueryRequestInfo { private final Boolean useQueryCache; private final Boolean useLegacySql; private final JobCreationMode jobCreationMode; + private final DataFormatOptions formatOptions; - QueryRequestInfo(QueryJobConfiguration config) { + QueryRequestInfo(QueryJobConfiguration config, Boolean useInt64Timestamps) { this.config = config; this.connectionProperties = config.getConnectionProperties(); this.defaultDataset = config.getDefaultDataset(); @@ -58,6 +60,7 @@ final class QueryRequestInfo { this.useLegacySql = config.useLegacySql(); this.useQueryCache = config.useQueryCache(); this.jobCreationMode = config.getJobCreationMode(); + this.formatOptions = new DataFormatOptions().setUseInt64Timestamp(useInt64Timestamps); } boolean isFastQuerySupported(JobId jobId) { @@ -122,6 +125,9 @@ QueryRequest toPb() { if (jobCreationMode != null) { request.setJobCreationMode(jobCreationMode.toString()); } + if (formatOptions != null) { + request.setFormatOptions(formatOptions); + } return request; } @@ -141,6 +147,7 @@ public String toString() { .add("useQueryCache", useQueryCache) .add("useLegacySql", useLegacySql) .add("jobCreationMode", jobCreationMode) + .add("formatOptions", formatOptions.getUseInt64Timestamp()) .toString(); } @@ -159,7 +166,8 @@ public int hashCode() { createSession, useQueryCache, useLegacySql, - jobCreationMode); + jobCreationMode, + formatOptions); } @Override diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java index 7d10a9750..5ade7c229 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueListTest.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import com.google.api.client.util.Data; @@ -52,6 +53,12 @@ public class FieldValueListTest { Field.of("tenth", LegacySQLTypeName.NUMERIC), Field.of("eleventh", LegacySQLTypeName.BIGNUMERIC)); + private final FieldList schemaLosslessTimestamp = + FieldList.of( + Field.of("first", LegacySQLTypeName.BOOLEAN), + Field.of("second", LegacySQLTypeName.INTEGER), + Field.of("third", LegacySQLTypeName.TIMESTAMP)); + private final Map integerPb = ImmutableMap.of("v", "1"); private final Map floatPb = ImmutableMap.of("v", "1.5"); private final Map stringPb = ImmutableMap.of("v", "string"); @@ -68,10 +75,15 @@ public class FieldValueListTest { "v", "99999999999999999999999999999999999999.99999999999999999999999999999999999999"); private final FieldValue booleanFv = FieldValue.of(Attribute.PRIMITIVE, "false"); + private final FieldValue booleanLosslessTimestampFv = + FieldValue.of(Attribute.PRIMITIVE, "false", true); private final FieldValue integerFv = FieldValue.of(Attribute.PRIMITIVE, "1"); + private final FieldValue integerLosslessTimestampFv = + FieldValue.of(Attribute.PRIMITIVE, "1", true); private final FieldValue floatFv = FieldValue.of(Attribute.PRIMITIVE, "1.5"); private final FieldValue stringFv = FieldValue.of(Attribute.PRIMITIVE, "string"); private final FieldValue timestampFv = FieldValue.of(Attribute.PRIMITIVE, "42"); + private final FieldValue losslessTimestampFv = FieldValue.of(Attribute.PRIMITIVE, "42", true); private final FieldValue bytesFv = FieldValue.of(Attribute.PRIMITIVE, BYTES_BASE64); private final FieldValue nullFv = FieldValue.of(Attribute.PRIMITIVE, null); private final FieldValue repeatedFv = @@ -117,11 +129,25 @@ public class FieldValueListTest { bigNumericFv), schema); + private final List fieldValuesLosslessTimestampPb = + ImmutableList.of(booleanPb, integerPb, timestampPb); + private final FieldValueList fieldValuesLosslessTimestamp = + FieldValueList.of( + ImmutableList.of( + booleanLosslessTimestampFv, integerLosslessTimestampFv, losslessTimestampFv), + schemaLosslessTimestamp); + @Test public void testFromPb() { assertEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, schema)); // Schema does not influence values equality assertEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, null)); + + assertNotEquals(fieldValues, FieldValueList.fromPb(fieldValuesPb, null, true)); + + assertEquals( + fieldValuesLosslessTimestamp, + FieldValueList.fromPb(fieldValuesLosslessTimestampPb, null, true)); } @Test diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java index 6dbe81707..4ec527f7c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FieldValueTest.java @@ -132,6 +132,25 @@ public void testTimestamp() { assertEquals(expected, received); } + @Test + public void testInt64Timestamp() { + FieldValue lossyFieldValue = + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "1.9954383398377106E10"); + long lossy = lossyFieldValue.getTimestampValue(); + + FieldValue losslessFieldValue = + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", true); + long lossless = losslessFieldValue.getTimestampValue(); + + assertEquals(lossy, lossless); + + FieldValue fieldValue = + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "19954383398377106", true); + long received = fieldValue.getTimestampValue(); + long expected = 19954383398377106L; + assertEquals(expected, received); + } + @Test public void testEquals() { FieldValue booleanValue = FieldValue.of(FieldValue.Attribute.PRIMITIVE, "false"); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java index 0d9464c76..420c01484 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java @@ -17,7 +17,9 @@ package com.google.cloud.bigquery; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import com.google.api.services.bigquery.model.QueryRequest; import com.google.cloud.bigquery.JobInfo.CreateDisposition; @@ -136,7 +138,7 @@ public class QueryRequestInfoTest { .setMaxResults(100L) .setJobCreationMode(jobCreationModeRequired) .build(); - QueryRequestInfo REQUEST_INFO = new QueryRequestInfo(QUERY_JOB_CONFIGURATION); + QueryRequestInfo REQUEST_INFO = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false); private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION_SUPPORTED = QueryJobConfiguration.newBuilder(QUERY) .setUseQueryCache(USE_QUERY_CACHE) @@ -150,7 +152,8 @@ public class QueryRequestInfoTest { .setCreateSession(CREATE_SESSION) .setMaxResults(100L) .build(); - QueryRequestInfo REQUEST_INFO_SUPPORTED = new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED); + QueryRequestInfo REQUEST_INFO_SUPPORTED = + new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false); @Test public void testIsFastQuerySupported() { @@ -171,8 +174,19 @@ public void testToPb() { @Test public void equalTo() { compareQueryRequestInfo( - new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED), REQUEST_INFO_SUPPORTED); - compareQueryRequestInfo(new QueryRequestInfo(QUERY_JOB_CONFIGURATION), REQUEST_INFO); + new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false), REQUEST_INFO_SUPPORTED); + compareQueryRequestInfo(new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false), REQUEST_INFO); + } + + @Test + public void testInt64Timestamp() { + QueryRequestInfo requestInfo = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false); + QueryRequest requestPb = requestInfo.toPb(); + assertFalse(requestPb.getFormatOptions().getUseInt64Timestamp()); + + QueryRequestInfo requestInfoLosslessTs = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, true); + QueryRequest requestLosslessTsPb = requestInfoLosslessTs.toPb(); + assertTrue(requestLosslessTsPb.getFormatOptions().getUseInt64Timestamp()); } /* @@ -199,5 +213,6 @@ private void compareQueryRequestInfo(QueryRequestInfo expected, QueryRequestInfo assertEquals(expectedQueryReq.getUseQueryCache(), actualQueryReq.getUseQueryCache()); assertEquals(expectedQueryReq.getUseLegacySql(), actualQueryReq.getUseLegacySql()); assertEquals(expectedQueryReq.get("jobCreationMode"), actualQueryReq.get("jobCreationMode")); + assertEquals(expectedQueryReq.getFormatOptions(), actualQueryReq.getFormatOptions()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 86ccd7c7d..84944d49c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -3235,6 +3235,42 @@ public void testTimestamp() throws InterruptedException { } } + @Test + public void testLosslessTimestamp() throws InterruptedException { + String query = "SELECT TIMESTAMP '2022-01-24T23:54:25.095574Z'"; + long expectedTimestamp = 1643068465095574L; + + TableResult result = + bigquery.query( + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DatasetId.of(DATASET)) + .build()); + assertNotNull(result.getJobId()); + for (FieldValueList row : result.getValues()) { + FieldValue timeStampCell = row.get(0); + assertFalse(timeStampCell.getUseInt64Timestamps()); + assertEquals(expectedTimestamp, timeStampCell.getTimestampValue()); + } + + // Create new BQ object to toggle lossless timestamps without affecting + // other tests. + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQuery bigqueryLossless = bigqueryHelper.getOptions().getService(); + bigqueryLossless.getOptions().setUseInt64Timestamps(true); + + TableResult resultLossless = + bigqueryLossless.query( + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DatasetId.of(DATASET)) + .build()); + assertNotNull(resultLossless.getJobId()); + for (FieldValueList row : resultLossless.getValues()) { + FieldValue timeStampCellLossless = row.get(0); + assertTrue(timeStampCellLossless.getUseInt64Timestamps()); + assertEquals(expectedTimestamp, timeStampCellLossless.getTimestampValue()); + } + } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQuery() throws InterruptedException {