From fd3975cd60843ede55adc81d4f0e7711a4af0ea8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 15:54:13 -0400 Subject: [PATCH 01/18] Revert "feat: Add high precision TIMESTAMP values for queries (#7147)" This reverts commit bea42b2d48ef9a53395c0ad585f3baa941f5f822. # Conflicts: # handwritten/bigquery/src/bigquery.ts # handwritten/bigquery/src/job.ts # handwritten/bigquery/test/bigquery.ts --- handwritten/bigquery/src/bigquery.ts | 85 ++------- handwritten/bigquery/src/job.ts | 27 +-- handwritten/bigquery/system-test/bigquery.ts | 164 +----------------- .../system-test/timestamp_output_format.ts | 30 ++-- handwritten/bigquery/test/bigquery.ts | 68 +------- 5 files changed, 40 insertions(+), 334 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index d52d07343b5..9bafee94e23 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -1100,11 +1100,6 @@ export class BigQuery extends Service { }; }), }; - } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { - return { - type: 'TIMESTAMP', - timestampPrecision: '12', - }; } providedType = (providedType as string).toUpperCase(); @@ -2256,33 +2251,14 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const listParams = { - 'formatOptions.timestampOutputFormat': - queryReq.formatOptions?.timestampOutputFormat, - 'formatOptions.useInt64Timestamp': - queryReq.formatOptions?.useInt64Timestamp, - }; - if (options.skipParsing) { - rows = res.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - listParams, - }); - delete res.rows; - } - } catch (e) { - (callback as SimpleQueryRowsCallback)(e as Error, null, job); - return; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + }); + delete res.rows; } } this.trace_('[runJobsQuery] job complete'); @@ -2364,18 +2340,6 @@ export class BigQuery extends Service { if (options.job) { return undefined; } - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - timestampOutputFormat: 'ISO8601_STRING', - }; - const formatOptions = extend(defaultOpts, { - timestampOutputFormat: options['formatOptions.timestampOutputFormat'], - useInt64Timestamp: options['formatOptions.useInt64Timestamp'], - }); const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2384,7 +2348,9 @@ export class BigQuery extends Service { maximumBytesBilled: queryObj.maximumBytesBilled, timeoutMs: options.timeoutMs, location: queryObj.location || options.location, - formatOptions, + formatOptions: { + useInt64Timestamp: true, + }, maxResults: queryObj.maxResults || options.maxResults, query: queryObj.query, useLegacySql: false, @@ -2628,7 +2594,6 @@ function convertSchemaFieldValue( value = BigQueryRange.fromSchemaValue_( value, schemaField.rangeElementType!.type!, - options.listParams, // Required to convert TIMESTAMP values ); break; } @@ -2706,14 +2671,6 @@ export class BigQueryRange { }; } - /** - * This method returns start and end values for RANGE typed values returned from - * the server. It decodes the server RANGE value into start and end values so - * they can be used to construct a BigQueryRange. - * @private - * @param {string} value The range value. - * @returns {string[]} The start and end of the range. - */ private static fromStringValue_(value: string): [start: string, end: string] { let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { @@ -2733,24 +2690,7 @@ export class BigQueryRange { return [start, end]; } - /** - * This method is only used by convertSchemaFieldValue and only when range - * values are passed into convertSchemaFieldValue. It produces a value that is - * delivered to the user for read calls and it needs to pass along listParams - * to ensure TIMESTAMP types are converted properly. - * @private - * @param {string} value The range value. - * @param {string} elementType The element type. - * @param {bigquery.tabledata.IListParams | bigquery.jobs.IGetQueryResultsParams} [listParams] The list parameters. - * @returns {BigQueryRange} - */ - static fromSchemaValue_( - value: string, - elementType: string, - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams, - ): BigQueryRange { + static fromSchemaValue_(value: string, elementType: string): BigQueryRange { const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { @@ -2758,7 +2698,6 @@ export class BigQueryRange { } return convertSchemaFieldValue({type: elementType}, value, { wrapIntegers: false, - listParams, }); }; return BigQuery.range( diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index 22554a1d8f3..dc0b068c2ad 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,25 +596,14 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - try { - /* - Without this try/catch block, calls to /query endpoint will hang - indefinitely if a call to mergeSchemaWithRows_ fails because the - error never makes it to the callback. Instead, pass the error to the - callback the user provides so that the user can see the error. - */ - if (options.skipParsing) { - rows = resp.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; - } - } catch (e) { - callback!(e as Error, null, null, resp); - return; + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; } } diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 92550709de1..c74f4ba959b 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1495,14 +1495,9 @@ describe('BigQuery', () => { ], }, (err, rows) => { - try { - // Without this try block the test runner silently fails - assert.ifError(err); - assert.strictEqual(rows!.length, 1); - done(); - } catch (e) { - done(e); - } + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); }, ); }); @@ -1526,159 +1521,6 @@ describe('BigQuery', () => { }, ); }); - describe('High Precision Query System Tests', () => { - let bigquery: BigQuery; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = - '2023-01-01T12:00:00.123456789123Z'; - const expectedErrorMessage = - 'Cannot specify both timestamp_as_int and timestamp_output_format.'; - - before(() => { - bigquery = new BigQuery(); - }); - - const testCases = [ - { - name: 'TOF: FLOAT64, UI64: true (error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedTsValue: undefined, - expectedError: expectedErrorMessage, - }, - { - name: 'TOF: omitted, UI64: omitted (default INT64)', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, - }, - { - name: 'TOF: omitted, UI64: true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. Queries using TIMESTAMP_ADD are another example. - */ - const query = { - query: 'SELECT ? as ts', - params: [ - bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), - ], - types: ['TIMESTAMP(12)'], - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].ts.value !== undefined); - assert.strictEqual( - rows[0].ts.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - it(`should handle nested ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. - */ - const query = { - query: 'SELECT ? obj', - params: [ - { - nested: { - a: bigquery.timestamp( - '2023-01-01T12:00:00.123456789123Z', - ), - }, - }, - ], - types: [ - { - nested: { - a: 'TIMESTAMP(12)', - }, - }, - ], - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].obj.nested.a.value !== undefined); - assert.strictEqual( - rows[0].obj.nested.a.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - }); - }); }); describe('named', () => { diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 0fe388e1e3b..96ede116075 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -34,8 +34,8 @@ describe('Timestamp Output Format System Tests', () => { const dataset = bigquery.dataset(datasetId); const table = dataset.table(tableId); const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; + const expectedTsValueMicroseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { await dataset.create(); @@ -59,13 +59,13 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', @@ -78,19 +78,19 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=true', timestampOutputFormat: 'INT64', useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=false', timestampOutputFormat: 'INT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', @@ -103,50 +103,50 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: false, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, // Additional test cases for undefined combinations { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', timestampOutputFormat: undefined, useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', timestampOutputFormat: undefined, useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', timestampOutputFormat: undefined, useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp undefined', timestampOutputFormat: 'INT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, ]; diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index 74fbbedf1c7..bc006e70660 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -42,7 +42,6 @@ import { TableField, Query, QueryResultsOptions, - QueryOptions, } from '../src'; import {SinonStub} from 'sinon'; import {PreciseDate} from '@google-cloud/precise-date'; @@ -3466,75 +3465,12 @@ describe('BigQuery', () => { }, jobCreationMode: 'JOB_CREATION_REQUIRED', formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, }, }; assert.deepStrictEqual(req, expectedReq); }); - describe('timestamp format options', () => { - const testCases: { - name: string; - opts: QueryOptions; - expected?: any; - bail?: boolean; - }[] = [ - { - name: 'TOF: omitted, UI64: omitted (default ISO8601_STRING)', - opts: {}, - expected: { - timestampOutputFormat: 'ISO8601_STRING', - }, - }, - { - name: 'TOF: omitted, UI64: true', - opts: { - ['formatOptions.useInt64Timestamp']: true, - }, - expected: { - useInt64Timestamp: true, - }, - }, - { - name: 'TOF: omitted, UI64: false (default ISO8601_STRING)', - opts: { - ['formatOptions.useInt64Timestamp']: false, - }, - expected: { - useInt64Timestamp: false, - }, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, () => { - const req = bq.buildQueryRequest_(QUERY_STRING, testCase.opts); - - const expectedReq = { - query: QUERY_STRING, - useLegacySql: false, - requestId: req.requestId, - jobCreationMode: 'JOB_CREATION_OPTIONAL', - formatOptions: testCase.expected, - connectionProperties: undefined, - continuous: undefined, - createSession: undefined, - defaultDataset: undefined, - destinationEncryptionConfiguration: undefined, - labels: undefined, - location: undefined, - maxResults: undefined, - maximumBytesBilled: undefined, - preserveNulls: undefined, - reservation: undefined, - timeoutMs: undefined, - useQueryCache: undefined, - writeIncrementalResults: undefined, - }; - assert.deepStrictEqual(req, expectedReq); - }); - }); - }); it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3548,7 +3484,7 @@ describe('BigQuery', () => { requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, }, }; assert.deepStrictEqual(req, expectedReq); From f31ccc7638afd5fcb88324015b27227f054b2ef3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:15:22 -0400 Subject: [PATCH 02/18] Revert "feat: support high precision timestamp strings on getRows calls (#1596)" This reverts commit b3217a39f0d85faaf606b97b85e0166d5f7fb07f. # Conflicts: # handwritten/bigquery/src/table.ts --- handwritten/bigquery/src/bigquery.ts | 56 +---- handwritten/bigquery/src/table.ts | 50 ++-- .../system-test/timestamp_output_format.ts | 216 ------------------ handwritten/bigquery/test/table.ts | 9 +- 4 files changed, 24 insertions(+), 307 deletions(-) delete mode 100644 handwritten/bigquery/system-test/timestamp_output_format.ts diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 9bafee94e23..116434f173d 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -597,9 +597,6 @@ export class BigQuery extends Service { wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams; }, ) { // deep copy schema fields to avoid mutation @@ -2480,9 +2477,6 @@ function convertSchemaFieldValue( wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams; }, ) { if (value === null) { @@ -2542,43 +2536,9 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - /* - At this point, 'value' will equal the timestamp value returned from the - server. We need to parse this value differently depending on its format. - For example, value could be any of the following: - 1672574400123456 - 1672574400.123456 - 2023-01-01T12:00:00.123456789123Z - */ - const listParams = options.listParams; - const timestampOutputFormat = listParams - ? listParams['formatOptions.timestampOutputFormat'] - : undefined; - const useInt64Timestamp = listParams - ? listParams['formatOptions.useInt64Timestamp'] - : undefined; - if (timestampOutputFormat === 'ISO8601_STRING') { - // value is ISO string, create BigQueryTimestamp wrapping the string - value = BigQuery.timestamp(value); - } else if ( - useInt64Timestamp !== true && - timestampOutputFormat !== 'INT64' && - (useInt64Timestamp !== undefined || timestampOutputFormat !== undefined) - ) { - // NOTE: The additional - // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) - // check is to ensure that calls to the /query endpoint remain - // unaffected as they will not be providing any listParams. - // - // If the program reaches this point in time then - // value is float seconds so convert to BigQueryTimestamp - value = BigQuery.timestamp(Number(value)); - } else { - // Expect int64 micros (default or explicit INT64) - const pd = new PreciseDate(); - pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); - value = BigQuery.timestamp(pd); - } + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); break; } case 'GEOGRAPHY': { @@ -2773,16 +2733,6 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index e92c6a6791d..62ed4521256 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -55,7 +55,6 @@ import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; import {IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; -import IDataFormatOptions = bigquery.IDataFormatOptions; // This is supposed to be a @google-cloud/storage `File` type. The storage npm // module includes these types, but is current installed as a devDependency. @@ -1867,42 +1866,25 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - if (options.skipParsing) { - rows = rows || []; - } else { - rows = BigQuery.mergeSchemaWithRows_( - this.metadata.schema, - rows || [], - { - wrapIntegers, - selectedFields, - parseJSON, - listParams: qs, - }, - ); - } - } catch (err) { - callback!(err as Error | null, null, null, resp); - return; + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], { + wrapIntegers, + selectedFields, + parseJSON, + }); } callback!(null, rows, nextQuery, resp); }; - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', - }; - const qs = extend(defaultOpts, options); + + const qs = extend( + { + 'formatOptions.useInt64Timestamp': true, + }, + options, + ); + this.request( { uri: '/data', diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts deleted file mode 100644 index 96ede116075..00000000000 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as assert from 'assert'; -import {describe, it, before, after} from 'mocha'; -import {BigQuery} from '../src/bigquery'; -import {randomUUID} from 'crypto'; -import {RequestResponse} from '@google-cloud/common/build/src/service-object'; - -const bigquery = new BigQuery(); - -interface TestCase { - name: string; - timestampOutputFormat?: string; - useInt64Timestamp?: boolean; - expectedError?: string; - expectedTsValue?: string; -} - -describe('Timestamp Output Format System Tests', () => { - const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; - const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; - const dataset = bigquery.dataset(datasetId); - const table = dataset.table(tableId); - const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; - const expectedTsValueMicroseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456789123Z'; - - before(async () => { - await dataset.create(); - await table.create({ - schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], - }); - // Insert a row to test retrieval - await table.insert([{ts: insertedTsValue}]); - }); - - after(async () => { - try { - await dataset.delete({force: true}); - } catch (e) { - console.error('Error deleting dataset:', e); - } - }); - - const testCases: TestCase[] = [ - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedError: - 'Cannot specify both use_int64_timestamp and timestamp_output_format.', - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp=true', - timestampOutputFormat: 'INT64', - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp=false', - timestampOutputFormat: 'INT64', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: true, - expectedError: - 'Cannot specify both use_int64_timestamp and timestamp_output_format.', - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, - }, - // Additional test cases for undefined combinations - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, - }, - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', - timestampOutputFormat: undefined, - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp undefined', - timestampOutputFormat: 'INT64', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach( - ({ - name, - timestampOutputFormat, - useInt64Timestamp, - expectedError, - expectedTsValue, - }) => { - it(name, async () => { - const options: {[key: string]: any} = {}; - if (timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - timestampOutputFormat; - } - if (useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = useInt64Timestamp; - } - - if (expectedError) { - try { - await table.getRows(options); - assert.fail('The call should have thrown an error.'); - } catch (e) { - assert.strictEqual((e as Error).message, expectedError); - } - } else { - const [rows] = await table.getRows(options); - assert(rows.length > 0); - assert.strictEqual(rows[0].ts.value, expectedTsValue); - } - }); - }, - ); - - it('should make a request with ISO8601_STRING when no format options are being used', done => { - void (async () => { - const originalRequest = table.request; - const requestPromise: Promise = new Promise( - (resolve, reject) => { - const innerPromise = new Promise((innerResolve, innerReject) => { - innerResolve({}); - }); - resolve(innerPromise as Promise); - }, - ); - table.request = reqOpts => { - table.request = originalRequest; - if ( - reqOpts.qs['formatOptions.timestampOutputFormat'] === 'ISO8601_STRING' - ) { - done(); - } else { - done( - new Error( - 'The default timestampOutputFormat should be ISO8601_STRING', - ), - ); - } - return requestPromise; - }; - await table.getRows({}); - })(); - }); -}); diff --git a/handwritten/bigquery/test/table.ts b/handwritten/bigquery/test/table.ts index 2e13f1572d6..bb3225459ea 100644 --- a/handwritten/bigquery/test/table.ts +++ b/handwritten/bigquery/test/table.ts @@ -2046,7 +2046,7 @@ describe('BigQuery/Table', () => { assert.strictEqual(reqOpts.uri, '/data'); assert.deepStrictEqual(reqOpts.qs, { ...options, - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; @@ -2219,6 +2219,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { callback(null, { + 'formatOptions.useInt64Timestamp': true, pageToken, }); }; @@ -2228,7 +2229,7 @@ describe('BigQuery/Table', () => { assert.deepStrictEqual(nextQuery, { a: 'b', c: 'd', - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, pageToken, }); // Original object isn't affected. @@ -2445,7 +2446,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; @@ -2469,7 +2470,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; From 5544e36556739a18cc37974cb7bb1a61f72846c8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:18:57 -0400 Subject: [PATCH 03/18] skip a test --- handwritten/bigquery/system-test/bigquery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index c74f4ba959b..58009c5e668 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1041,7 +1041,7 @@ describe('BigQuery', () => { } }); - it('should create a table with timestampPrecision', async () => { + it.skip('should create a table with timestampPrecision', async () => { const table = dataset.table(generateName('timestamp-precision-table')); const schema = { fields: [ From 1bbd192f0fcbbc1b6eea6bac51c1f47de0aebd1b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:28:18 -0400 Subject: [PATCH 04/18] Remove the High Precision Query System tests --- handwritten/bigquery/system-test/bigquery.ts | 155 ------------------- 1 file changed, 155 deletions(-) diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 58009c5e668..aced74c4b40 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1815,161 +1815,6 @@ describe('BigQuery', () => { }, ); }); - describe('High Precision Query System Tests', () => { - let bigquery: BigQuery; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = - '2023-01-01T12:00:00.123456789123Z'; - const expectedErrorMessage = - 'Cannot specify both timestamp_as_int and timestamp_output_format.'; - - before(() => { - bigquery = new BigQuery(); - }); - - const testCases = [ - { - name: 'TOF: FLOAT64, UI64: true (error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedTsValue: undefined, - expectedError: expectedErrorMessage, - }, - { - name: 'TOF: omitted, UI64: omitted (default INT64)', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, - }, - { - name: 'TOF: omitted, UI64: true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. Queries using TIMESTAMP_ADD are another example. - */ - const query = { - query: 'SELECT @r AS ts', - params: { - r: bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), - }, - types: { - r: 'TIMESTAMP(12)', - }, - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].ts.value !== undefined); - assert.strictEqual( - rows[0].ts.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - it(`should handle nested ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. - */ - const query = { - query: 'SELECT @r obj', - params: { - r: { - nested: { - a: bigquery.timestamp( - '2023-01-01T12:00:00.123456789123Z', - ), - }, - }, - }, - types: { - r: { - nested: { - a: 'TIMESTAMP(12)', - }, - }, - }, - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].obj.nested.a.value !== undefined); - assert.strictEqual( - rows[0].obj.nested.a.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - }); - }); }); }); From c588794d657711f4b95294cc2ea46b287f89a697 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:21:22 -0400 Subject: [PATCH 05/18] Source code changes for timestamp precision flag --- handwritten/bigquery/src/bigquery.ts | 152 ++++++++++++++++++++++++--- handwritten/bigquery/src/job.ts | 37 +++++-- handwritten/bigquery/src/table.ts | 73 ++++++++++--- 3 files changed, 228 insertions(+), 34 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 116434f173d..010bea0e105 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -597,6 +597,9 @@ export class BigQuery extends Service { wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams; }, ) { // deep copy schema fields to avoid mutation @@ -1097,6 +1100,13 @@ export class BigQuery extends Service { }; }), }; + } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + return { + type: 'TIMESTAMP', + timestampPrecision: '12', + }; + } } providedType = (providedType as string).toUpperCase(); @@ -2248,14 +2258,45 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - if (options.skipParsing) { - rows = res.rows; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listParams = { + 'formatOptions.timestampOutputFormat': + queryReq.formatOptions?.timestampOutputFormat, + 'formatOptions.useInt64Timestamp': + queryReq.formatOptions?.useInt64Timestamp, + }; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + listParams, + }); + delete res.rows; + } + } catch (e) { + (callback as SimpleQueryRowsCallback)(e as Error, null, job); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - }); - delete res.rows; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + }); + delete res.rows; + } } } this.trace_('[runJobsQuery] job complete'); @@ -2337,6 +2378,25 @@ export class BigQuery extends Service { if (options.job) { return undefined; } + let formatOptions; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + timestampOutputFormat: 'ISO8601_STRING', + }; + formatOptions = extend(defaultOpts, { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }); + } else { + formatOptions = { + useInt64Timestamp: true, + }; + } const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2345,9 +2405,7 @@ export class BigQuery extends Service { maximumBytesBilled: queryObj.maximumBytesBilled, timeoutMs: options.timeoutMs, location: queryObj.location || options.location, - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, maxResults: queryObj.maxResults || options.maxResults, query: queryObj.query, useLegacySql: false, @@ -2477,6 +2535,9 @@ function convertSchemaFieldValue( wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams; }, ) { if (value === null) { @@ -2536,9 +2597,52 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - const pd = new PreciseDate(); - pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); - value = BigQuery.timestamp(pd); + // High precision timestamp behaviour + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + /* + At this point, 'value' will equal the timestamp value returned from the + server. We need to parse this value differently depending on its format. + For example, value could be any of the following: + 1672574400123456 + 1672574400.123456 + 2023-01-01T12:00:00.123456789123Z + */ + const listParams = options.listParams; + const timestampOutputFormat = listParams + ? listParams['formatOptions.timestampOutputFormat'] + : undefined; + const useInt64Timestamp = listParams + ? listParams['formatOptions.useInt64Timestamp'] + : undefined; + if (timestampOutputFormat === 'ISO8601_STRING') { + // value is ISO string, create BigQueryTimestamp wrapping the string + value = BigQuery.timestamp(value); + } else if ( + useInt64Timestamp !== true && + timestampOutputFormat !== 'INT64' && + (useInt64Timestamp !== undefined || + timestampOutputFormat !== undefined) + ) { + // NOTE: The additional + // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) + // check is to ensure that calls to the /query endpoint remain + // unaffected as they will not be providing any listParams. + // + // If the program reaches this point in time then + // value is float seconds so convert to BigQueryTimestamp + value = BigQuery.timestamp(Number(value)); + } else { + // Expect int64 micros (default or explicit INT64) + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); + } + } else { + // Old behaviour + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); + } break; } case 'GEOGRAPHY': { @@ -2554,6 +2658,7 @@ function convertSchemaFieldValue( value = BigQueryRange.fromSchemaValue_( value, schemaField.rangeElementType!.type!, + options.listParams, // Required to convert TIMESTAMP values ); break; } @@ -2650,7 +2755,13 @@ export class BigQueryRange { return [start, end]; } - static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + static fromSchemaValue_( + value: string, + elementType: string, + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams, + ): BigQueryRange { const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { @@ -2658,6 +2769,7 @@ export class BigQueryRange { } return convertSchemaFieldValue({type: elementType}, value, { wrapIntegers: false, + listParams, }); }; return BigQuery.range( @@ -2733,6 +2845,18 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { + /* + TODO: + When https://github.com/googleapis/nodejs-precise-date/pull/302 + is released and we have full support for picoseconds in PreciseData + then we can remove this if block. + */ + this.value = value; + return; + } + } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index dc0b068c2ad..a0a2b45c1ad 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,14 +596,37 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - if (options.skipParsing) { - rows = resp.rows; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + try { + /* + Without this try/catch block, calls to /query endpoint will hang + indefinitely if a call to mergeSchemaWithRows_ fails because the + error never makes it to the callback. Instead, pass the error to the + callback the user provides so that the user can see the error. + */ + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; + } + } catch (e) { + callback!(e as Error, null, null, resp); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; + } } } diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index 62ed4521256..f425a21caef 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1866,24 +1866,71 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - if (options.skipParsing) { - rows = rows || []; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + // High precision timestamp behaviour + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_( + this.metadata.schema, + rows || [], + { + wrapIntegers, + selectedFields, + parseJSON, + listParams: qs, + }, + ); + } + } catch (err) { + callback!(err as Error | null, null, null, resp); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], { - wrapIntegers, - selectedFields, - parseJSON, - }); + // Old behaviour + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_( + this.metadata.schema, + rows || [], + { + wrapIntegers, + selectedFields, + parseJSON, + }, + ); + } } callback!(null, rows, nextQuery, resp); }; - const qs = extend( - { - 'formatOptions.useInt64Timestamp': true, - }, - options, - ); + let qs: any; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + }; + qs = extend(defaultOpts, options); + } else { + qs = extend( + { + 'formatOptions.useInt64Timestamp': true, + }, + options, + ); + } this.request( { From 73c4ed5f9d91b9a8c4c242738a5b2aa0edd52e63 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:36:50 -0400 Subject: [PATCH 06/18] Add tests back --- handwritten/bigquery/system-test/bigquery.ts | 381 ++++++++++++++++-- .../system-test/timestamp_output_format.ts | 0 2 files changed, 358 insertions(+), 23 deletions(-) create mode 100644 handwritten/bigquery/system-test/timestamp_output_format.ts diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index aced74c4b40..8a104d9d9db 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1041,26 +1041,28 @@ describe('BigQuery', () => { } }); - it.skip('should create a table with timestampPrecision', async () => { - const table = dataset.table(generateName('timestamp-precision-table')); - const schema = { - fields: [ - { - name: 'ts_field', - type: 'TIMESTAMP', - timestampPrecision: 12, - }, - ], - }; - try { - await table.create({schema}); - const [metadata] = await table.getMetadata(); - assert.deepStrictEqual( - metadata.schema.fields[0].timestampPrecision, - '12', - ); - } catch (e) { - assert.ifError(e); + it('should create a table with timestampPrecision', async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const table = dataset.table(generateName('timestamp-precision-table')); + const schema = { + fields: [ + { + name: 'ts_field', + type: 'TIMESTAMP', + timestampPrecision: 12, + }, + ], + }; + try { + await table.create({schema}); + const [metadata] = await table.getMetadata(); + assert.deepStrictEqual( + metadata.schema.fields[0].timestampPrecision, + '12', + ); + } catch (e) { + assert.ifError(e); + } } }); @@ -1495,9 +1497,14 @@ describe('BigQuery', () => { ], }, (err, rows) => { - assert.ifError(err); - assert.strictEqual(rows!.length, 1); - done(); + try { + // Without this try block the test runner silently fails + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); + } catch (e) { + done(e); + } }, ); }); @@ -1521,6 +1528,169 @@ describe('BigQuery', () => { }, ); }); + describe('High Precision Query System Tests', () => { + let bigquery: BigQuery; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = + '2023-01-01T12:00:00.123456789123Z'; + const expectedErrorMessage = + 'Cannot specify both timestamp_as_int and timestamp_output_format.'; + + before(() => { + bigquery = new BigQuery(); + }); + + const testCases = [ + { + name: 'TOF: FLOAT64, UI64: true (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedTsValue: undefined, + expectedError: expectedErrorMessage, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. Queries using TIMESTAMP_ADD are another example. + */ + const query = { + query: 'SELECT ? as ts', + params: [ + bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), + ], + types: ['TIMESTAMP(12)'], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].ts.value !== undefined); + assert.strictEqual( + rows[0].ts.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. + */ + const query = { + query: 'SELECT ? obj', + params: [ + { + nested: { + a: bigquery.timestamp( + '2023-01-01T12:00:00.123456789123Z', + ), + }, + }, + ], + types: [ + { + nested: { + a: 'TIMESTAMP(12)', + }, + }, + ], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].obj.nested.a.value !== undefined); + assert.strictEqual( + rows[0].obj.nested.a.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + }); + }); }); describe('named', () => { @@ -1815,6 +1985,171 @@ describe('BigQuery', () => { }, ); }); + describe('High Precision Query System Tests', () => { + let bigquery: BigQuery; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = + '2023-01-01T12:00:00.123456789123Z'; + const expectedErrorMessage = + 'Cannot specify both timestamp_as_int and timestamp_output_format.'; + + before(() => { + bigquery = new BigQuery(); + }); + + const testCases = [ + { + name: 'TOF: FLOAT64, UI64: true (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedTsValue: undefined, + expectedError: expectedErrorMessage, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. Queries using TIMESTAMP_ADD are another example. + */ + const query = { + query: 'SELECT @r AS ts', + params: { + r: bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), + }, + types: { + r: 'TIMESTAMP(12)', + }, + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].ts.value !== undefined); + assert.strictEqual( + rows[0].ts.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. + */ + const query = { + query: 'SELECT @r obj', + params: { + r: { + nested: { + a: bigquery.timestamp( + '2023-01-01T12:00:00.123456789123Z', + ), + }, + }, + }, + types: { + r: { + nested: { + a: 'TIMESTAMP(12)', + }, + }, + }, + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].obj.nested.a.value !== undefined); + assert.strictEqual( + rows[0].obj.nested.a.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + }); + }); }); }); diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts new file mode 100644 index 00000000000..e69de29bb2d From ab16690d8cfdedecec278597cddea98cdaf98893 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:37:17 -0400 Subject: [PATCH 07/18] Add test file back in --- .../system-test/timestamp_output_format.ts | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index e69de29bb2d..0fe388e1e3b 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it, before, after} from 'mocha'; +import {BigQuery} from '../src/bigquery'; +import {randomUUID} from 'crypto'; +import {RequestResponse} from '@google-cloud/common/build/src/service-object'; + +const bigquery = new BigQuery(); + +interface TestCase { + name: string; + timestampOutputFormat?: string; + useInt64Timestamp?: boolean; + expectedError?: string; + expectedTsValue?: string; +} + +describe('Timestamp Output Format System Tests', () => { + const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; + const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; + const dataset = bigquery.dataset(datasetId); + const table = dataset.table(tableId); + const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; + + before(async () => { + await dataset.create(); + await table.create({ + schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], + }); + // Insert a row to test retrieval + await table.insert([{ts: insertedTsValue}]); + }); + + after(async () => { + try { + await dataset.delete({force: true}); + } catch (e) { + console.error('Error deleting dataset:', e); + } + }); + + const testCases: TestCase[] = [ + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedError: + 'Cannot specify both use_int64_timestamp and timestamp_output_format.', + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp=true', + timestampOutputFormat: 'INT64', + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp=false', + timestampOutputFormat: 'INT64', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, + expectedError: + 'Cannot specify both use_int64_timestamp and timestamp_output_format.', + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: false, + expectedTsValue: expectedTsValuePicoseconds, + }, + // Additional test cases for undefined combinations + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', + timestampOutputFormat: undefined, + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp undefined', + timestampOutputFormat: 'INT64', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + ]; + + testCases.forEach( + ({ + name, + timestampOutputFormat, + useInt64Timestamp, + expectedError, + expectedTsValue, + }) => { + it(name, async () => { + const options: {[key: string]: any} = {}; + if (timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + timestampOutputFormat; + } + if (useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = useInt64Timestamp; + } + + if (expectedError) { + try { + await table.getRows(options); + assert.fail('The call should have thrown an error.'); + } catch (e) { + assert.strictEqual((e as Error).message, expectedError); + } + } else { + const [rows] = await table.getRows(options); + assert(rows.length > 0); + assert.strictEqual(rows[0].ts.value, expectedTsValue); + } + }); + }, + ); + + it('should make a request with ISO8601_STRING when no format options are being used', done => { + void (async () => { + const originalRequest = table.request; + const requestPromise: Promise = new Promise( + (resolve, reject) => { + const innerPromise = new Promise((innerResolve, innerReject) => { + innerResolve({}); + }); + resolve(innerPromise as Promise); + }, + ); + table.request = reqOpts => { + table.request = originalRequest; + if ( + reqOpts.qs['formatOptions.timestampOutputFormat'] === 'ISO8601_STRING' + ) { + done(); + } else { + done( + new Error( + 'The default timestampOutputFormat should be ISO8601_STRING', + ), + ); + } + return requestPromise; + }; + await table.getRows({}); + })(); + }); +}); From 2410d66976e17a8c2d9679e759c9142f7e5534c0 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:20:22 -0400 Subject: [PATCH 08/18] Modify tests to only run for high precision or not --- handwritten/bigquery/test/bigquery.ts | 92 +++++++++++++++++++++++++-- handwritten/bigquery/test/table.ts | 71 ++++++++++++++++----- 2 files changed, 142 insertions(+), 21 deletions(-) diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index bc006e70660..e5bbf0e222a 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -42,6 +42,7 @@ import { TableField, Query, QueryResultsOptions, + QueryOptions, } from '../src'; import {SinonStub} from 'sinon'; import {PreciseDate} from '@google-cloud/precise-date'; @@ -3438,6 +3439,14 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, @@ -3464,13 +3473,78 @@ describe('BigQuery', () => { key: 'value', }, jobCreationMode: 'JOB_CREATION_REQUIRED', - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); + describe('timestamp format options', () => { + const testCases: { + name: string; + opts: QueryOptions; + expected?: any; + bail?: boolean; + }[] = [ + { + name: 'TOF: omitted, UI64: omitted (default ISO8601_STRING)', + opts: {}, + expected: { + timestampOutputFormat: 'ISO8601_STRING', + }, + }, + { + name: 'TOF: omitted, UI64: true', + opts: { + ['formatOptions.useInt64Timestamp']: true, + }, + expected: { + useInt64Timestamp: true, + }, + }, + { + name: 'TOF: omitted, UI64: false (default ISO8601_STRING)', + opts: { + ['formatOptions.useInt64Timestamp']: false, + }, + expected: { + useInt64Timestamp: false, + }, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } + const req = bq.buildQueryRequest_(QUERY_STRING, testCase.opts); + + const expectedReq = { + query: QUERY_STRING, + useLegacySql: false, + requestId: req.requestId, + jobCreationMode: 'JOB_CREATION_OPTIONAL', + formatOptions: testCase.expected, + connectionProperties: undefined, + continuous: undefined, + createSession: undefined, + defaultDataset: undefined, + destinationEncryptionConfiguration: undefined, + labels: undefined, + location: undefined, + maxResults: undefined, + maximumBytesBilled: undefined, + preserveNulls: undefined, + reservation: undefined, + timeoutMs: undefined, + useQueryCache: undefined, + writeIncrementalResults: undefined, + }; + assert.deepStrictEqual(req, expectedReq); + }); + }); + }); + it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3478,14 +3552,20 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); diff --git a/handwritten/bigquery/test/table.ts b/handwritten/bigquery/test/table.ts index bb3225459ea..f3e59d7f3d9 100644 --- a/handwritten/bigquery/test/table.ts +++ b/handwritten/bigquery/test/table.ts @@ -2041,12 +2041,20 @@ describe('BigQuery/Table', () => { it('should make correct API request', done => { const options = {a: 'b', c: 'd'}; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.strictEqual(reqOpts.uri, '/data'); assert.deepStrictEqual(reqOpts.qs, { ...options, - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; @@ -2201,13 +2209,16 @@ describe('BigQuery/Table', () => { sandbox.restore(); const mergeStub = sandbox.stub(BigQuery, 'mergeSchemaWithRows_'); - table.getRows({skipParsing: true}, (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { - assert.ifError(err); - assert.strictEqual(rows_, rows); - assert.strictEqual(mergeStub.called, false); - assert.deepStrictEqual(apiResponse.rows, rows); - done(); - }); + table.getRows( + {skipParsing: true}, + (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { + assert.ifError(err); + assert.strictEqual(rows_, rows); + assert.strictEqual(mergeStub.called, false); + assert.deepStrictEqual(apiResponse.rows, rows); + done(); + }, + ); }); it('should pass nextQuery if pageToken is returned', done => { @@ -2217,19 +2228,33 @@ describe('BigQuery/Table', () => { // Set a schema so it doesn't try to refresh the metadata. table.metadata = {schema: {}}; + const callbackResponse = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.useInt64Timestamp': true, + pageToken, + } + : { + pageToken, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { - callback(null, { - 'formatOptions.useInt64Timestamp': true, - pageToken, - }); + callback(null, callbackResponse); }; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.getRows(options, (err: Error, rows: {}, nextQuery: {}) => { assert.ifError(err); assert.deepStrictEqual(nextQuery, { a: 'b', c: 'd', - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, pageToken, }); // Original object isn't affected. @@ -2443,10 +2468,18 @@ describe('BigQuery/Table', () => { const wrapIntegers = {integerTypeCastFunction: sinon.stub()}; const options = {wrapIntegers}; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; @@ -2467,10 +2500,18 @@ describe('BigQuery/Table', () => { parseJSON: true, }; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; From bdba4ddf8b4455c29d1d025e12902e7a56f9e9c9 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:28:20 -0400 Subject: [PATCH 09/18] listParams add --- handwritten/bigquery/src/bigquery.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 010bea0e105..187a8ea0762 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2294,6 +2294,7 @@ export class BigQuery extends Service { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, + listParams, }); delete res.rows; } From ca0a51cd0edb2cfdb4421ad9e03dab5cb18b1fae Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:46:20 -0400 Subject: [PATCH 10/18] remove listParams --- handwritten/bigquery/src/bigquery.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 187a8ea0762..010bea0e105 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2294,7 +2294,6 @@ export class BigQuery extends Service { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, - listParams, }); delete res.rows; } From a1eb4d20d3ef5c111d1b253be99a0a5a95a2de47 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:58:59 -0400 Subject: [PATCH 11/18] skip tests if picosecond support is not turned on --- .../bigquery/system-test/timestamp_output_format.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 0fe388e1e3b..c12cddb7d7d 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -28,7 +28,7 @@ interface TestCase { expectedTsValue?: string; } -describe('Timestamp Output Format System Tests', () => { +describe.only('Timestamp Output Format System Tests', () => { const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; const dataset = bigquery.dataset(datasetId); @@ -159,6 +159,9 @@ describe('Timestamp Output Format System Tests', () => { expectedTsValue, }) => { it(name, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } const options: {[key: string]: any} = {}; if (timestampOutputFormat !== undefined) { options['formatOptions.timestampOutputFormat'] = @@ -185,6 +188,10 @@ describe('Timestamp Output Format System Tests', () => { ); it('should make a request with ISO8601_STRING when no format options are being used', done => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + done(); + return; + } void (async () => { const originalRequest = table.request; const requestPromise: Promise = new Promise( From a959b3ac379029012124e404c71423d48dc1253c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:59:52 -0400 Subject: [PATCH 12/18] remove only --- handwritten/bigquery/system-test/timestamp_output_format.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index c12cddb7d7d..9c3325db632 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -28,7 +28,7 @@ interface TestCase { expectedTsValue?: string; } -describe.only('Timestamp Output Format System Tests', () => { +describe('Timestamp Output Format System Tests', () => { const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; const dataset = bigquery.dataset(datasetId); From f5b7e3199ba8cc9c0741249df6e4d4abfe44cc7c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 13:41:03 -0400 Subject: [PATCH 13/18] =?UTF-8?q?Don=E2=80=99t=20create=20separate=20code?= =?UTF-8?q?=20paths=20when=20not=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handwritten/bigquery/src/bigquery.ts | 48 +++++++++++----------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 010bea0e105..b11b3afa7a3 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2258,45 +2258,33 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const listParams = { - 'formatOptions.timestampOutputFormat': - queryReq.formatOptions?.timestampOutputFormat, - 'formatOptions.useInt64Timestamp': - queryReq.formatOptions?.useInt64Timestamp, - }; - if (options.skipParsing) { - rows = res.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - listParams, - }); - delete res.rows; - } - } catch (e) { - (callback as SimpleQueryRowsCallback)(e as Error, null, job); - return; - } - } else { + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listParams = { + 'formatOptions.timestampOutputFormat': + queryReq.formatOptions?.timestampOutputFormat, + 'formatOptions.useInt64Timestamp': + queryReq.formatOptions?.useInt64Timestamp, + }; if (options.skipParsing) { rows = res.rows; } else { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, + listParams, }); delete res.rows; } + } catch (e) { + (callback as SimpleQueryRowsCallback)(e as Error, null, job); + return; } } this.trace_('[runJobsQuery] job complete'); From 61db403eff7cb8778659d5821458d3cde279a90b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 14:08:31 -0400 Subject: [PATCH 14/18] extend the default options --- handwritten/bigquery/src/bigquery.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index b11b3afa7a3..fbfcf2bd907 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2381,9 +2381,15 @@ export class BigQuery extends Service { useInt64Timestamp: options['formatOptions.useInt64Timestamp'], }); } else { - formatOptions = { - useInt64Timestamp: true, - }; + formatOptions = extend( + { + useInt64Timestamp: true, + }, + { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }, + ); } const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, From 01ab527f4c1d4b892cc9111aa0bf5918c689d46f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 14:09:39 -0400 Subject: [PATCH 15/18] Skip the before hook if picosecond support not on --- handwritten/bigquery/system-test/timestamp_output_format.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 9c3325db632..bd622b9dfec 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -38,6 +38,9 @@ describe('Timestamp Output Format System Tests', () => { const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } await dataset.create(); await table.create({ schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], From 67a48221d5433051d12bdf3e0d1664b7e32c7783 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:28:22 -0400 Subject: [PATCH 16/18] Add documentation back in for methods --- handwritten/bigquery/src/bigquery.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index fbfcf2bd907..f13a491ac4b 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2730,6 +2730,14 @@ export class BigQueryRange { }; } + /** + * This method returns start and end values for RANGE typed values returned from + * the server. It decodes the server RANGE value into start and end values so + * they can be used to construct a BigQueryRange. + * @private + * @param {string} value The range value. + * @returns {string[]} The start and end of the range. + */ private static fromStringValue_(value: string): [start: string, end: string] { let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { @@ -2749,6 +2757,17 @@ export class BigQueryRange { return [start, end]; } + /** + * This method is only used by convertSchemaFieldValue and only when range + * values are passed into convertSchemaFieldValue. It produces a value that is + * delivered to the user for read calls and it needs to pass along listParams + * to ensure TIMESTAMP types are converted properly. + * @private + * @param {string} value The range value. + * @param {string} elementType The element type. + * @param {bigquery.tabledata.IListParams | bigquery.jobs.IGetQueryResultsParams} [listParams] The list parameters. + * @returns {BigQueryRange} + */ static fromSchemaValue_( value: string, elementType: string, From 757dfdcf8ad78b2bea8272e3ef1513f9bf0f3053 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:39:48 -0400 Subject: [PATCH 17/18] Remove the if block since release is done --- handwritten/bigquery/src/bigquery.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index f13a491ac4b..23f1ec680bd 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2858,18 +2858,6 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } - } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { From cbeeacc199289a9e20cc0edefd348bab4745c0d3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:46:17 -0400 Subject: [PATCH 18/18] Eliminate redundant feature flag check. --- handwritten/bigquery/src/job.ts | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index a0a2b45c1ad..22554a1d8f3 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,28 +596,13 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - try { - /* - Without this try/catch block, calls to /query endpoint will hang - indefinitely if a call to mergeSchemaWithRows_ fails because the - error never makes it to the callback. Instead, pass the error to the - callback the user provides so that the user can see the error. - */ - if (options.skipParsing) { - rows = resp.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; - } - } catch (e) { - callback!(e as Error, null, null, resp); - return; - } - } else { + try { + /* + Without this try/catch block, calls to /query endpoint will hang + indefinitely if a call to mergeSchemaWithRows_ fails because the + error never makes it to the callback. Instead, pass the error to the + callback the user provides so that the user can see the error. + */ if (options.skipParsing) { rows = resp.rows; } else { @@ -627,6 +612,9 @@ class Job extends Operation { }); delete resp.rows; } + } catch (e) { + callback!(e as Error, null, null, resp); + return; } }