test: reproduce commonly cited timeout error

mock-server/datastore-server.ts

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// eslint-disable-next-line n/no-extraneous-import
+import {ServiceError} from 'google-gax';
+import {Server} from '@grpc/grpc-js';
+
 const {dirname, resolve} = require('node:path');
 
 const PROTO_PATH = __dirname + '/../protos/google/datastore/v1/datastore.proto';
@@ -40,22 +44,43 @@ const descriptor = grpc.loadPackageDefinition(packageDefinition);
 /**
  * Implements the runQuery RPC method.
  */
-function grpcEndpoint(
-  call: {},
-  callback: (arg1: string | null, arg2: {}) => {},
+function grpcEndpoint<ResponseType>(
+  call: CallType,
+  callback: MockServiceCallback<ResponseType>,
 ) {
   // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST.
-  callback(null, {message: 'Hello'});
+  callback(null, {message: 'Hello'} as ResponseType);
+}
+
+export type CallType = () => {};
+export type GrpcErrorType = ServiceError | null;
+
+type MockServiceCallback<ResponseType> = (
+  arg1: GrpcErrorType,
+  arg2: ResponseType,
+) => {};
+
+interface MockServiceConfiguration<ResponseType> {
+  [endpoint: string]: (
+    call: CallType,
+    callback: MockServiceCallback<ResponseType>,
+  ) => void;
 }
 
 /**
  * Starts an RPC server that receives requests for datastore
  */
-export function startServer(cb: () => void) {
+export function startServer<ResponseType>(
+  cb: () => void,
+  serviceConfigurationOverride?: MockServiceConfiguration<ResponseType>,
+): Server {
   const server = new grpc.Server();
   const service = descriptor.google.datastore.v1.Datastore.service;
   // On the next line, change runQuery to the grpc method you want to investigate
-  server.addService(service, {runQuery: grpcEndpoint});
+  const serviceConfiguration = serviceConfigurationOverride ?? {
+    runQuery: grpcEndpoint,
+  };
+  server.addService(service, serviceConfiguration);
   server.bindAsync(
     '0.0.0.0:50051',
     grpc.ServerCredentials.createInsecure(),
@@ -64,4 +89,5 @@ export function startServer(cb: () => void) {
       cb();
     },
   );
+  return server;
 }

test/mock-server-tester.ts

@@ -0,0 +1,91 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {CallType, GrpcErrorType} from '../mock-server/datastore-server';
+
+import grpc = require('@grpc/grpc-js');
+import {ServiceError} from 'google-gax';
+
+export class ErrorGenerator {
+  private errorSeriesCount = 0;
+
+  /**
+   * Generates an error object for testing purposes.
+   *
+   * This method creates a `ServiceError` object, simulating an error
+   * that might be returned by a gRPC service. The error includes a
+   * `DEADLINE_EXCEEDED` code (4), a details message indicating the number of
+   * errors generated so far by this instance, and some metadata.
+   *
+   * @returns {ServiceError} A `ServiceError` object representing a simulated
+   *   gRPC error.
+   */
+  generateError(code: number) {
+    // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST.
+    this.errorSeriesCount++;
+    const metadata = new grpc.Metadata();
+    metadata.set(
+      'grpc-server-stats-bin',
+      Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]),
+    );
+    const error = new Error('error message') as ServiceError;
+    error.code = code;
+    error.details = `error details: error count: ${this.errorSeriesCount}`;
+    error.metadata = metadata;
+    return error;
+  }
+
+  /**
+   * Returns a function that simulates an error response from a gRPC service.
+   *
+   * This method is designed to be used in mock server setups for testing purposes.
+   * It returns a function that, when called, will invoke a callback with a
+   * pre-configured error object, simulating a gRPC service responding with an error.
+   * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the
+   * number of errors generated so far by this instance, and some metadata.
+   *
+   * @param {number} code The grpc error code for the error sent back
+   * @returns {function} A function that takes a `call` object (representing the
+   *   gRPC call) and a `callback` function, and responds to the call with a
+   *   simulated error.
+   */
+  sendErrorSeries<ResponseType>(code: number) {
+    return (
+      call: CallType,
+      callback: (arg1: GrpcErrorType, arg2: ResponseType) => {},
+    ) => {
+      const error = this.generateError(code);
+      callback(error, {} as ResponseType);
+    };
+  }
+}
+
+export function shutdownServer(server: grpc.Server) {
+  return new Promise((resolve, reject) => {
+    // Assuming 'server.tryShutdown' is a function that takes a callback.
+    // The callback is expected to be called when the shutdown attempt is complete.
+    // If 'tryShutdown' itself can throw an error or indicate an immediate failure
+    // (without calling the callback), you might need to wrap this call in a try...catch block.
+
+    server.tryShutdown((error: Error | undefined) => {
+      if (error) {
+        // If the callback is called with an error, reject the promise.
+        reject(error);
+      } else {
+        // If the callback is called without an error, resolve the promise.
+        resolve('done');
+      }
+    });
+  });
+}

test/with-runquery-mockserver.ts

@@ -0,0 +1,66 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {describe, it} from 'mocha';
+import {Datastore} from '../src';
+import * as assert from 'assert';
+
+import {startServer} from '../mock-server/datastore-server';
+import {ErrorGenerator, shutdownServer} from './mock-server-tester';
+import {grpc} from 'google-gax';
+
+describe('Should make calls to runQuery', () => {
+  it('should report an UNAVAILABLE error to the user when it occurs with the original error details', done => {
+    const errorGenerator = new ErrorGenerator();
+    const server = startServer(
+      async () => {
+        try {
+          try {
+            const datastore = new Datastore({
+              namespace: `${Date.now()}`,
+              apiEndpoint: 'localhost:50051',
+              projectId: 'test-project', // Provided to avoid 'Unable to detect a Project Id in the current environment.'
+            });
+            const postKey = datastore.key(['Post', 'post1']);
+            const query = datastore.createQuery('Post').hasAncestor(postKey);
+            // Make the call with a shorter timeout:
+            await datastore.runQuery(query, {gaxOptions: {timeout: 5000}});
+            assert.fail('The call should not have succeeded');
+          } catch (e) {
+            const message = (e as Error).message;
+            const searchValue = /[.*+?^${}()|[\]\\]/g;
+            const replaceValue = '\\$&';
+            const substringToFind1 =
+              'Total timeout of API google.datastore.v1.Datastore exceeded 5000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:';
+            const escapedSubstringRegex1 = new RegExp(
+              substringToFind1.replace(searchValue, replaceValue),
+            );
+            assert.match(message, escapedSubstringRegex1);
+            const substringToFind2 =
+              'before any response was received. : Previous errors : [{message: 14 UNAVAILABLE: error details: error count: 1, code: 14, details: , note: },{message: 14 UNAVAILABLE: error details: error count: 2, code: 14, details: , note: },';
+            const escapedSubstringRegex2 = new RegExp(
+              substringToFind2.replace(searchValue, replaceValue),
+            );
+            assert.match(message, escapedSubstringRegex2);
+            done();
+          }
+        } catch (e) {
+          done(e);
+        }
+        await shutdownServer(server);
+      },
+      {runQuery: errorGenerator.sendErrorSeries(grpc.status.UNAVAILABLE)},
+    );
+  });
+});