Skip to content

Commit bea42b2

Browse files
danieljbrucegoogle-labs-jules[bot]alvarowolfx
authored
feat: Add high precision TIMESTAMP values for queries (#7147)
* move over all code changes supporting high precision timestamps * feat: Add high precision TIMESTAMP values for queries * correct typo in comment * picoseconds / not nanoseconds * rename nanoseconds to picoseconds * nanoseconds not microseconds * Add comment for non-meaningful use case * feat(bigquery): add unit tests for buildQueryRequest_ format options Adds a new test suite to bigquery.ts that verifies the correct construction of QueryRequest formatOptions for various combinations of timestampOutputFormat and useInt64Timestamp. The test cases are modeled after the high-precision-query system tests. Co-authored-by: danieljbruce <8935272+danieljbruce@users.noreply.github.com> * Move the code back into the BigQueryRange static fns * Move the “High Precision Query System Tests” block * Revert "Move the “High Precision Query System Tests” block" This reverts commit f042cc4. * Reapply "Move the “High Precision Query System Tests” block" This reverts commit 8c4c2d5. * Move the tests to the end position * delete unwanted test cases * Change the name used in the describe block * Apply suggestion from @alvarowolfx Co-authored-by: Alvaro Viebrantz <aviebrantz@google.com> * Linter and clean up last commit * Fix unit tests in test/bigquery to expect right va * JS doc conversions * remove only * Remove bail: true * removed todo * remove error in name * remove error from tests --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> Co-authored-by: danieljbruce <8935272+danieljbruce@users.noreply.github.com> Co-authored-by: Alvaro Viebrantz <aviebrantz@google.com>
1 parent c1630b5 commit bea42b2

File tree

5 files changed

+443
-33
lines changed

5 files changed

+443
-33
lines changed

‎handwritten/bigquery/src/bigquery.ts‎

Lines changed: 70 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,11 @@ export class BigQuery extends Service {
10991099
};
11001100
}),
11011101
};
1102+
} else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') {
1103+
return {
1104+
type: 'TIMESTAMP',
1105+
timestampPrecision: '12',
1106+
};
11021107
}
11031108

11041109
providedType = (providedType as string).toUpperCase();
@@ -2249,11 +2254,30 @@ export class BigQuery extends Service {
22492254
if (res && res.jobComplete) {
22502255
let rows: any = [];
22512256
if (res.schema && res.rows) {
2252-
rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, {
2253-
wrapIntegers: options.wrapIntegers || false,
2254-
parseJSON: options.parseJSON,
2255-
});
2256-
delete res.rows;
2257+
try {
2258+
/*
2259+
Without this try/catch block, calls to getRows will hang indefinitely if
2260+
a call to mergeSchemaWithRows_ fails because the error never makes it to
2261+
the callback. Instead, pass the error to the callback the user provides
2262+
so that the user can see the error.
2263+
*/
2264+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
2265+
const listParams = {
2266+
'formatOptions.timestampOutputFormat':
2267+
queryReq.formatOptions?.timestampOutputFormat,
2268+
'formatOptions.useInt64Timestamp':
2269+
queryReq.formatOptions?.useInt64Timestamp,
2270+
};
2271+
rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, {
2272+
wrapIntegers: options.wrapIntegers || false,
2273+
parseJSON: options.parseJSON,
2274+
listParams,
2275+
});
2276+
delete res.rows;
2277+
} catch (e) {
2278+
(callback as SimpleQueryRowsCallback)(e as Error, null, job);
2279+
return;
2280+
}
22572281
}
22582282
this.trace_('[runJobsQuery] job complete');
22592283
options._cachedRows = rows;
@@ -2334,6 +2358,18 @@ export class BigQuery extends Service {
23342358
if (options.job) {
23352359
return undefined;
23362360
}
2361+
const hasAnyFormatOpts =
2362+
options['formatOptions.timestampOutputFormat'] !== undefined ||
2363+
options['formatOptions.useInt64Timestamp'] !== undefined;
2364+
const defaultOpts = hasAnyFormatOpts
2365+
? {}
2366+
: {
2367+
timestampOutputFormat: 'ISO8601_STRING',
2368+
};
2369+
const formatOptions = extend(defaultOpts, {
2370+
timestampOutputFormat: options['formatOptions.timestampOutputFormat'],
2371+
useInt64Timestamp: options['formatOptions.useInt64Timestamp'],
2372+
});
23372373
const req: bigquery.IQueryRequest = {
23382374
useQueryCache: queryObj.useQueryCache,
23392375
labels: queryObj.labels,
@@ -2342,9 +2378,7 @@ export class BigQuery extends Service {
23422378
maximumBytesBilled: queryObj.maximumBytesBilled,
23432379
timeoutMs: options.timeoutMs,
23442380
location: queryObj.location || options.location,
2345-
formatOptions: {
2346-
useInt64Timestamp: true,
2347-
},
2381+
formatOptions,
23482382
maxResults: queryObj.maxResults || options.maxResults,
23492383
query: queryObj.query,
23502384
useLegacySql: false,
@@ -2588,6 +2622,7 @@ function convertSchemaFieldValue(
25882622
value = BigQueryRange.fromSchemaValue_(
25892623
value,
25902624
schemaField.rangeElementType!.type!,
2625+
options.listParams, // Required to convert TIMESTAMP values
25912626
);
25922627
break;
25932628
}
@@ -2665,6 +2700,14 @@ export class BigQueryRange {
26652700
};
26662701
}
26672702

2703+
/**
2704+
* This method returns start and end values for RANGE typed values returned from
2705+
* the server. It decodes the server RANGE value into start and end values so
2706+
* they can be used to construct a BigQueryRange.
2707+
* @private
2708+
* @param {string} value The range value.
2709+
* @returns {string[]} The start and end of the range.
2710+
*/
26682711
private static fromStringValue_(value: string): [start: string, end: string] {
26692712
let cleanedValue = value;
26702713
if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) {
@@ -2684,14 +2727,32 @@ export class BigQueryRange {
26842727
return [start, end];
26852728
}
26862729

2687-
static fromSchemaValue_(value: string, elementType: string): BigQueryRange {
2730+
/**
2731+
* This method is only used by convertSchemaFieldValue and only when range
2732+
* values are passed into convertSchemaFieldValue. It produces a value that is
2733+
* delivered to the user for read calls and it needs to pass along listParams
2734+
* to ensure TIMESTAMP types are converted properly.
2735+
* @private
2736+
* @param {string} value The range value.
2737+
* @param {string} elementType The element type.
2738+
* @param {bigquery.tabledata.IListParams | bigquery.jobs.IGetQueryResultsParams} [listParams] The list parameters.
2739+
* @returns {BigQueryRange}
2740+
*/
2741+
static fromSchemaValue_(
2742+
value: string,
2743+
elementType: string,
2744+
listParams?:
2745+
| bigquery.tabledata.IListParams
2746+
| bigquery.jobs.IGetQueryResultsParams,
2747+
): BigQueryRange {
26882748
const [start, end] = BigQueryRange.fromStringValue_(value);
26892749
const convertRangeSchemaValue = (value: string) => {
26902750
if (value === 'UNBOUNDED' || value === 'NULL') {
26912751
return null;
26922752
}
26932753
return convertSchemaFieldValue({type: elementType}, value, {
26942754
wrapIntegers: false,
2755+
listParams,
26952756
});
26962757
};
26972758
return BigQuery.range(

‎handwritten/bigquery/src/job.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -595,10 +595,21 @@ class Job extends Operation {
595595
let rows: any = [];
596596

597597
if (resp.schema && resp.rows) {
598-
rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, {
599-
wrapIntegers,
600-
parseJSON,
601-
});
598+
try {
599+
/*
600+
Without this try/catch block, calls to /query endpoint will hang
601+
indefinitely if a call to mergeSchemaWithRows_ fails because the
602+
error never makes it to the callback. Instead, pass the error to the
603+
callback the user provides so that the user can see the error.
604+
*/
605+
rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, {
606+
wrapIntegers,
607+
parseJSON,
608+
});
609+
} catch (e) {
610+
callback!(e as Error, null, null, resp);
611+
return;
612+
}
602613
}
603614

604615
let nextQuery: QueryResultsOptions | null = null;

‎handwritten/bigquery/system-test/bigquery.ts‎

Lines changed: 161 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1472,9 +1472,14 @@ describe('BigQuery', () => {
14721472
],
14731473
},
14741474
(err, rows) => {
1475-
assert.ifError(err);
1476-
assert.strictEqual(rows!.length, 1);
1477-
done();
1475+
try {
1476+
// Without this try block the test runner silently fails
1477+
assert.ifError(err);
1478+
assert.strictEqual(rows!.length, 1);
1479+
done();
1480+
} catch (e) {
1481+
done(e);
1482+
}
14781483
},
14791484
);
14801485
});
@@ -1498,6 +1503,159 @@ describe('BigQuery', () => {
14981503
},
14991504
);
15001505
});
1506+
describe('High Precision Query System Tests', () => {
1507+
let bigquery: BigQuery;
1508+
const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z';
1509+
const expectedTsValuePicoseconds =
1510+
'2023-01-01T12:00:00.123456789123Z';
1511+
const expectedErrorMessage =
1512+
'Cannot specify both timestamp_as_int and timestamp_output_format.';
1513+
1514+
before(() => {
1515+
bigquery = new BigQuery();
1516+
});
1517+
1518+
const testCases = [
1519+
{
1520+
name: 'TOF: FLOAT64, UI64: true (error)',
1521+
timestampOutputFormat: 'FLOAT64',
1522+
useInt64Timestamp: true,
1523+
expectedTsValue: undefined,
1524+
expectedError: expectedErrorMessage,
1525+
},
1526+
{
1527+
name: 'TOF: omitted, UI64: omitted (default INT64)',
1528+
timestampOutputFormat: undefined,
1529+
useInt64Timestamp: undefined,
1530+
expectedTsValue: expectedTsValuePicoseconds,
1531+
},
1532+
{
1533+
name: 'TOF: omitted, UI64: true',
1534+
timestampOutputFormat: undefined,
1535+
useInt64Timestamp: true,
1536+
expectedTsValue: expectedTsValueNanoseconds,
1537+
},
1538+
];
1539+
1540+
testCases.forEach(testCase => {
1541+
it(`should handle ${testCase.name}`, async () => {
1542+
/*
1543+
The users use the new TIMESTAMP(12) type to indicate they want to
1544+
opt in to using timestampPrecision=12. The reason is that some queries
1545+
like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set
1546+
timestampPrecision=12 and we don't want this code change to affect
1547+
existing users. Queries using TIMESTAMP_ADD are another example.
1548+
*/
1549+
const query = {
1550+
query: 'SELECT ? as ts',
1551+
params: [
1552+
bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'),
1553+
],
1554+
types: ['TIMESTAMP(12)'],
1555+
};
1556+
1557+
const options: any = {};
1558+
if (testCase.timestampOutputFormat !== undefined) {
1559+
options['formatOptions.timestampOutputFormat'] =
1560+
testCase.timestampOutputFormat;
1561+
}
1562+
if (testCase.useInt64Timestamp !== undefined) {
1563+
options['formatOptions.useInt64Timestamp'] =
1564+
testCase.useInt64Timestamp;
1565+
}
1566+
1567+
try {
1568+
const [rows] = await bigquery.query(query, options);
1569+
if (testCase.expectedError) {
1570+
assert.fail(
1571+
`Query should have failed for ${testCase.name}, but succeeded`,
1572+
);
1573+
}
1574+
assert.ok(rows.length > 0);
1575+
assert.ok(rows[0].ts.value !== undefined);
1576+
assert.strictEqual(
1577+
rows[0].ts.value,
1578+
testCase.expectedTsValue,
1579+
);
1580+
} catch (err: any) {
1581+
if (!testCase.expectedError) {
1582+
throw err;
1583+
}
1584+
1585+
const message = err.message;
1586+
assert.strictEqual(
1587+
message,
1588+
testCase.expectedError,
1589+
`Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`,
1590+
);
1591+
}
1592+
});
1593+
it(`should handle nested ${testCase.name}`, async () => {
1594+
/*
1595+
The users use the new TIMESTAMP(12) type to indicate they want to
1596+
opt in to using timestampPrecision=12. The reason is that some queries
1597+
like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set
1598+
timestampPrecision=12 and we don't want this code change to affect
1599+
existing users.
1600+
*/
1601+
const query = {
1602+
query: 'SELECT ? obj',
1603+
params: [
1604+
{
1605+
nested: {
1606+
a: bigquery.timestamp(
1607+
'2023-01-01T12:00:00.123456789123Z',
1608+
),
1609+
},
1610+
},
1611+
],
1612+
types: [
1613+
{
1614+
nested: {
1615+
a: 'TIMESTAMP(12)',
1616+
},
1617+
},
1618+
],
1619+
};
1620+
1621+
const options: any = {};
1622+
if (testCase.timestampOutputFormat !== undefined) {
1623+
options['formatOptions.timestampOutputFormat'] =
1624+
testCase.timestampOutputFormat;
1625+
}
1626+
if (testCase.useInt64Timestamp !== undefined) {
1627+
options['formatOptions.useInt64Timestamp'] =
1628+
testCase.useInt64Timestamp;
1629+
}
1630+
1631+
try {
1632+
const [rows] = await bigquery.query(query, options);
1633+
if (testCase.expectedError) {
1634+
assert.fail(
1635+
`Query should have failed for ${testCase.name}, but succeeded`,
1636+
);
1637+
}
1638+
assert.ok(rows.length > 0);
1639+
assert.ok(rows[0].obj.nested.a.value !== undefined);
1640+
assert.strictEqual(
1641+
rows[0].obj.nested.a.value,
1642+
testCase.expectedTsValue,
1643+
);
1644+
} catch (err: any) {
1645+
if (!testCase.expectedError) {
1646+
throw err;
1647+
}
1648+
1649+
const message = err.message;
1650+
assert.strictEqual(
1651+
message,
1652+
testCase.expectedError,
1653+
`Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`,
1654+
);
1655+
}
1656+
});
1657+
});
1658+
});
15011659
});
15021660

15031661
describe('named', () => {

0 commit comments

Comments
 (0)