From ff86989af630a8bf37943586c24228d3dcf5459b Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 23 Apr 2024 15:41:57 +0200 Subject: [PATCH] RUBY-3390 CSOT for lisCollections --- lib/mongo/bulk_write.rb | 46 +- lib/mongo/collection/view/iterable.rb | 2 +- lib/mongo/collection/view/readable.rb | 29 +- lib/mongo/database.rb | 6 + lib/mongo/database/view.rb | 26 +- lib/mongo/index/view.rb | 16 +- lib/mongo/operation/find/op_msg.rb | 4 +- lib/mongo/retryable/read_worker.rb | 9 +- spec/runners/unified/crud_operations.rb | 37 +- spec/runners/unified/ddl_operations.rb | 10 + .../retryability-legacy-timeouts.yml | 1676 ++++++++++ .../retryability-timeoutMS.yml | 2824 +++++++++++++++++ 12 files changed, 4635 insertions(+), 50 deletions(-) create mode 100644 spec/spec_tests/data/client_side_operations_timeout/retryability-legacy-timeouts.yml create mode 100644 spec/spec_tests/data/client_side_operations_timeout/retryability-timeoutMS.yml diff --git a/lib/mongo/bulk_write.rb b/lib/mongo/bulk_write.rb index 22559dbbef..1286e39634 100644 --- a/lib/mongo/bulk_write.rb +++ b/lib/mongo/bulk_write.rb @@ -60,20 +60,14 @@ def execute result_combiner = ResultCombiner.new operations = op_combiner.combine validate_requests! + deadline = calculate_deadline client.with_session(@options) do |session| operations.each do |operation| - op_timeout_ms = if @deadline - if @deadline == 0 - 0 - else - ((@deadline - Utils.monotonic_time) * 1_000).to_i - end - end context = Operation::Context.new( client: client, session: session, - operation_timeouts: { operation_timeout_ms: op_timeout_ms } + operation_timeouts: { operation_timeout_ms: op_timeout_ms(deadline) } ) if single_statement?(operation) write_concern = write_concern(session) @@ -135,15 +129,8 @@ def initialize(collection, requests, options = {}) @collection = collection @requests = requests @options = options || {} - if @options[:timeout_ms] - @timeout_ms = @options[:timeout_ms] - if @timeout_ms > 0 - @deadline = Utils.monotonic_time + ( @timeout_ms / 1_000.0 ) - elsif @timeout_ms == 0 - @deadline = 0 - else - raise ArgumentError, "timeout_ms options must be non-negative integer" - end + if @options[:timeout_ms] && @options[:timeout_ms] < 0 + raise ArgumentError, "timeout_ms options must be non-negative integer" end end @@ -183,6 +170,31 @@ def write_concern(session = nil) :update_one, :insert_one ].freeze + # @return [ Float | nil ] Deadline for the batch of operations, if set. + def calculate_deadline + timeout_ms = @options[:timeout_ms] || collection.timeout_ms + return nil if timeout_ms.nil? + + if timeout_ms == 0 + 0 + else + Utils.monotonic_time + (timeout_ms / 1_000.0) + end + end + + # @param [ Float | nil ] deadline Deadline for the batch of operations. + # + # @return [ Integer | nil ] Timeout in milliseconds for the next operation. + def op_timeout_ms(deadline) + return nil if deadline.nil? + + if deadline == 0 + 0 + else + ((deadline - Utils.monotonic_time) * 1_000).to_i + end + end + def single_statement?(operation) SINGLE_STATEMENT_OPS.include?(operation.keys.first) end diff --git a/lib/mongo/collection/view/iterable.rb b/lib/mongo/collection/view/iterable.rb index a391f82471..c35a23d559 100644 --- a/lib/mongo/collection/view/iterable.rb +++ b/lib/mongo/collection/view/iterable.rb @@ -99,7 +99,7 @@ def select_cursor(session) Cursor.new(view, result, server, session: session, context: context) end else - read_with_retry_cursor(session, server_selector, view) do |server| + read_with_retry_cursor(session, server_selector, view, context: context) do |server| send_initial_query(server, context) end end diff --git a/lib/mongo/collection/view/readable.rb b/lib/mongo/collection/view/readable.rb index c1b22680c5..d548031ac4 100644 --- a/lib/mongo/collection/view/readable.rb +++ b/lib/mongo/collection/view/readable.rb @@ -187,7 +187,7 @@ def count(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - read_with_retry(session, selector) do |server| + read_with_retry(session, selector, context) do |server| Operation::Count.new( selector: cmd, db_name: database.name, @@ -237,7 +237,7 @@ def count_documents(opts = {}) pipeline << { :'$limit' => opts[:limit] } if opts[:limit] pipeline << { :'$group' => { _id: 1, n: { :'$sum' => 1 } } } - opts = opts.slice(:hint, :max_time_ms, :read, :collation, :session, :comment) + opts = opts.slice(:hint, :max_time_ms, :read, :collation, :session, :comment, :timeout_ms) opts[:collation] ||= collation first = aggregate(pipeline, opts).first @@ -279,12 +279,12 @@ def estimated_document_count(opts = {}) read_pref = opts[:read] || read_preference selector = ServerSelector.get(read_pref || server_selector) with_session(opts) do |session| - read_with_retry(session, selector) do |server| - context = Operation::Context.new( - client: client, - session: session, - operation_timeouts: operation_timeouts(opts) - ) + context = Operation::Context.new( + client: client, + session: session, + operation_timeouts: operation_timeouts(opts) + ) + read_with_retry(session, selector, context) do |server| cmd = { count: collection.name } cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms] if read_concern @@ -347,7 +347,12 @@ def distinct(field_name, opts = {}) read_pref = opts[:read] || read_preference selector = ServerSelector.get(read_pref || server_selector) with_session(opts) do |session| - read_with_retry(session, selector) do |server| + context = Operation::Context.new( + client: client, + session: session, + operation_timeouts: operation_timeouts(opts) + ) + read_with_retry(session, selector, context) do |server| Operation::Distinct.new( selector: cmd, db_name: database.name, @@ -360,11 +365,7 @@ def distinct(field_name, opts = {}) collation: opts[:collation] || opts['collation'] || collation, ).execute( server, - context: Operation::Context.new( - client: client, - session: session, - operation_timeouts: operation_timeouts(opts) - ) + context: context ) end.first['values'] end diff --git a/lib/mongo/database.rb b/lib/mongo/database.rb index db0742209c..97d3844e9c 100644 --- a/lib/mongo/database.rb +++ b/lib/mongo/database.rb @@ -128,6 +128,8 @@ def [](collection_name, options = {}) # required privilege to run the command when access control is enforced # @option options [ Object ] :comment A user-provided # comment to attach to this command. + # @option options [ Integer ] :timeout_ms The operation timeout in milliseconds. + # Must a positive integer. The default value is unset which means infinite. # # See https://mongodb.com/docs/manual/reference/command/listCollections/ # for more information and usage. @@ -156,6 +158,8 @@ def collection_names(options = {}) # required privilege to run the command when access control is enforced. # @option options [ Object ] :comment A user-provided # comment to attach to this command. + # @option options [ Integer ] :timeout_ms The operation timeout in milliseconds. + # Must a positive integer. The default value is unset which means infinite. # # See https://mongodb.com/docs/manual/reference/command/listCollections/ # for more information and usage. @@ -181,6 +185,8 @@ def list_collections(options = {}) # required privilege to run the command when access control is enforced. # @option options [ Object ] :comment A user-provided # comment to attach to this command. + # @option options [ Integer ] :timeout_ms The operation timeout in milliseconds. + # Must a positive integer. The default value is unset which means infinite. # # See https://mongodb.com/docs/manual/reference/command/listCollections/ # for more information and usage. diff --git a/lib/mongo/database/view.rb b/lib/mongo/database/view.rb index 73a8b0f420..28e4e38645 100644 --- a/lib/mongo/database/view.rb +++ b/lib/mongo/database/view.rb @@ -60,6 +60,8 @@ class View # to run the command when access control is enforced. # @option options [ Object ] :comment A user-provided # comment to attach to this command. + # @option options [ Integer ] :timeout_ms The operation timeout in milliseconds. + # Must a positive integer. The default value is unset which means infinite. # # See https://mongodb.com/docs/manual/reference/command/listCollections/ # for more information and usage. @@ -70,9 +72,14 @@ class View # @since 2.0.0 def collection_names(options = {}) @batch_size = options[:batch_size] - session = client.send(:get_session, options) - cursor = read_with_retry_cursor(session, ServerSelector.primary, self) do |server| - send_initial_query(server, session, options.merge(name_only: true)) + session = client.get_session(options) + context = Operation::Context.new( + client: client, + session: session, + operation_timeouts: operation_timeouts(options) + ) + cursor = read_with_retry_cursor(session, ServerSelector.primary, self, context: context) do |server| + send_initial_query(server, session, context, options.merge(name_only: true)) end cursor.map do |info| if cursor.initial_result.connection_description.features.list_collections_enabled? @@ -198,11 +205,16 @@ def operation_timeouts(opts = {}) def collections_info(session, server_selector, options = {}, &block) description = nil - cursor = read_with_retry_cursor(session, server_selector, self) do |server| + context = Operation::Context.new( + client: client, + session: session, + operation_timeouts: operation_timeouts(options) + ) + cursor = read_with_retry_cursor(session, server_selector, self, context: context) do |server| # TODO take description from the connection used to send the query # once https://jira.mongodb.org/browse/RUBY-1601 is fixed. description = server.description - send_initial_query(server, session, options) + send_initial_query(server, session, context, options) end # On 3.0+ servers, we get just the collection names. # On 2.6 server, we get collection names prefixed with the database @@ -266,7 +278,7 @@ def initial_query_op(session, options = {}) # types (where possible). # # @return [ Operation::Result ] Result of the query. - def send_initial_query(server, session, options = {}) + def send_initial_query(server, session, context, options = {}) opts = options.dup execution_opts = {} if opts.key?(:deserialize_as_bson) @@ -274,7 +286,7 @@ def send_initial_query(server, session, options = {}) end initial_query_op(session, opts).execute( server, - context: Operation::Context.new(client: client, session: session, operation_timeouts: operation_timeouts(opts)), + context: context, options: execution_opts ) end diff --git a/lib/mongo/index/view.rb b/lib/mongo/index/view.rb index c7bd5d6718..72fd13c698 100644 --- a/lib/mongo/index/view.rb +++ b/lib/mongo/index/view.rb @@ -273,9 +273,15 @@ def get(keys_or_name) # # @since 2.0.0 def each(&block) - session = client.send(:get_session, @options) - cursor = read_with_retry_cursor(session, ServerSelector.primary, self) do |server| - send_initial_query(server, session) + session = client.get_session(@options) + context = Operation::Context.new( + client: client, + session: session, + operation_timeouts: operation_timeouts(@options) + ) + + cursor = read_with_retry_cursor(session, ServerSelector.primary, self, context: context) do |server| + send_initial_query(server, session, context) end if block_given? cursor.each do |doc| @@ -386,8 +392,8 @@ def normalize_models(models, server) end end - def send_initial_query(server, session) - initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session)) + def send_initial_query(server, session, context) + initial_query_op(session).execute(server, context: context) end end end diff --git a/lib/mongo/operation/find/op_msg.rb b/lib/mongo/operation/find/op_msg.rb index 08e86f3f87..1cdb4baeb8 100644 --- a/lib/mongo/operation/find/op_msg.rb +++ b/lib/mongo/operation/find/op_msg.rb @@ -71,7 +71,9 @@ def apply_find_timeouts_to(spec, timeout_ms) end end - spec + spec.tap do |spc| + spc.delete(:maxTimeMS) if spc[:maxTimeMS].nil? + end end def selector(connection) diff --git a/lib/mongo/retryable/read_worker.rb b/lib/mongo/retryable/read_worker.rb index 4f3f14283f..0a1f437901 100644 --- a/lib/mongo/retryable/read_worker.rb +++ b/lib/mongo/retryable/read_worker.rb @@ -280,8 +280,6 @@ def read_without_retry(session, server_selector, &block) # # @return [ Result ] The result of the operation. def retry_read(original_error, session, server_selector, context: nil, failed_server: nil, &block) - context&.check_timeout! - server = select_server_for_retry( original_error, session, server_selector, context, failed_server ) @@ -289,11 +287,14 @@ def retry_read(original_error, session, server_selector, context: nil, failed_se log_retry(original_error, message: 'Read retry') begin + context&.check_timeout! attempt = attempt ? attempt + 1 : 2 yield server, true + rescue Error::TimeoutError + raise rescue *retryable_exceptions => e e.add_notes('modern retry', "attempt #{attempt}") - if context&.deadline + if context&.csot? failed_server = server retry else @@ -303,7 +304,7 @@ def retry_read(original_error, session, server_selector, context: nil, failed_se e.add_note('modern retry') if e.write_retryable? e.add_note("attempt #{attempt}") - if context&.deadline + if context&.csot? failed_server = server retry else diff --git a/spec/runners/unified/crud_operations.rb b/spec/runners/unified/crud_operations.rb index fd1730aea1..50a97f2cdc 100644 --- a/spec/runners/unified/crud_operations.rb +++ b/spec/runners/unified/crud_operations.rb @@ -47,6 +47,23 @@ def get_find_view(op) end end + def count(op) + collection = entities.get(:collection, op.use!('object')) + use_arguments(op) do |args| + opts = {} + if session = args.use('session') + opts[:session] = entities.get(:session, session) + end + if comment = args.use('comment') + opts[:comment] = comment + end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end + collection.count(args.use!('filter'), **opts) + end + end + def count_documents(op) collection = entities.get(:collection, op.use!('object')) use_arguments(op) do |args| @@ -57,6 +74,9 @@ def count_documents(op) if comment = args.use('comment') opts[:comment] = comment end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end collection.find(args.use!('filter')).count_documents(**opts) end end @@ -71,6 +91,9 @@ def estimated_document_count(op) if comment = args.use('comment') opts[:comment] = comment end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end collection.estimated_document_count(**opts) end end @@ -85,6 +108,9 @@ def distinct(op) if comment = args.use('comment') opts[:comment] = comment end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end req = collection.find(args.use!('filter'), **opts).distinct(args.use!('fieldName'), **opts) result = req.to_a end @@ -194,6 +220,7 @@ def update_one(op) comment: args.use('comment'), hint: args.use('hint'), upsert: args.use('upsert'), + timeout_ms: args.use('timeoutMS'), } if session = args.use('session') opts[:session] = entities.get(:session, session) @@ -209,6 +236,7 @@ def update_many(op) let: args.use('let'), comment: args.use('comment'), hint: args.use('hint'), + timeout_ms: args.use('timeoutMS'), } collection.update_many(args.use!('filter'), args.use!('update'), **opts) end @@ -224,7 +252,7 @@ def replace_one(op) upsert: args.use('upsert'), let: args.use('let'), hint: args.use('hint'), - timeout_ms: args.use('timeout_ms') + timeout_ms: args.use('timeoutMS') ) end end @@ -240,6 +268,9 @@ def delete_one(op) if session = args.use('session') opts[:session] = entities.get(:session, session) end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end collection.delete_one(args.use!('filter'), **opts) end end @@ -292,6 +323,10 @@ def aggregate(op) opts[:session] = entities.get(:session, session) end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end + unless args.empty? raise NotImplementedError, "Unhandled spec keys: #{args} in #{test_spec}" end diff --git a/spec/runners/unified/ddl_operations.rb b/spec/runners/unified/ddl_operations.rb index ede7f15647..134bd5ba1a 100644 --- a/spec/runners/unified/ddl_operations.rb +++ b/spec/runners/unified/ddl_operations.rb @@ -20,6 +20,9 @@ def list_dbs(op, name_only: false) if session = args.use('session') opts[:session] = entities.get(:session, session) end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end client.list_databases(args.use('filter') || {}, name_only, **opts) end end @@ -72,6 +75,10 @@ def list_colls(op, name_only: false) opts[:session] = entities.get(:session, session) end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end + database.list_collections(**opts.merge(name_only: name_only)) end end @@ -130,6 +137,9 @@ def list_indexes(op) if session = args.use('session') opts[:session] = entities.get(:session, session) end + if timeout_ms = args.use('timeoutMS') + opts[:timeout_ms] = timeout_ms + end collection.indexes(**opts).to_a end end diff --git a/spec/spec_tests/data/client_side_operations_timeout/retryability-legacy-timeouts.yml b/spec/spec_tests/data/client_side_operations_timeout/retryability-legacy-timeouts.yml new file mode 100644 index 0000000000..abcaec6127 --- /dev/null +++ b/spec/spec_tests/data/client_side_operations_timeout/retryability-legacy-timeouts.yml @@ -0,0 +1,1676 @@ +# Tests in this file are generated from retryability-legacy-timeouts.yml.template. + +description: "legacy timeouts behave correctly for retryable operations" + +schemaVersion: "1.9" + +runOnRequirements: + - minServerVersion: "4.4" + topologies: ["replicaset", "sharded"] + +createEntities: + - client: + id: &failPointClient failPointClient + useMultipleMongoses: false + - client: + id: &client client + uriOptions: + socketTimeoutMS: 100 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + ignoreCommandMonitoringEvents: + - killCursors + - database: + id: &database database + client: *client + databaseName: &databaseName test + - collection: + id: &collection collection + database: *database + collectionName: &collectionName coll + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: [] + +tests: + # For each retryable operation, run two tests: + # + # 1. Socket timeouts are retried once - Each test constructs a client entity with socketTimeoutMS=100, configures a + # fail point to block the operation once for 125ms, and expects the operation to succeed. + # + # 2. Operations fail after two consecutive socket timeouts - Same as (1) but the fail point is configured to block + # the operation twice and the test expects the operation to fail. + + - description: "operation succeeds after one socket timeout - insertOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: insertOne + object: *collection + arguments: + document: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation fails after two consecutive socket timeouts - insertOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: insertOne + object: *collection + arguments: + document: { x: 1 } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation succeeds after one socket timeout - insertMany on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: insertMany + object: *collection + arguments: + documents: + - { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation fails after two consecutive socket timeouts - insertMany on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: insertMany + object: *collection + arguments: + documents: + - { x: 1 } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation succeeds after one socket timeout - deleteOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["delete"] + blockConnection: true + blockTimeMS: 125 + - name: deleteOne + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + - description: "operation fails after two consecutive socket timeouts - deleteOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["delete"] + blockConnection: true + blockTimeMS: 125 + - name: deleteOne + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + - description: "operation succeeds after one socket timeout - replaceOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 125 + - name: replaceOne + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - description: "operation fails after two consecutive socket timeouts - replaceOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 125 + - name: replaceOne + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - description: "operation succeeds after one socket timeout - updateOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 125 + - name: updateOne + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - description: "operation fails after two consecutive socket timeouts - updateOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 125 + - name: updateOne + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + - description: "operation succeeds after one socket timeout - findOneAndDelete on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndDelete + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation fails after two consecutive socket timeouts - findOneAndDelete on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndDelete + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation succeeds after one socket timeout - findOneAndReplace on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndReplace + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation fails after two consecutive socket timeouts - findOneAndReplace on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndReplace + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation succeeds after one socket timeout - findOneAndUpdate on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndUpdate + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation fails after two consecutive socket timeouts - findOneAndUpdate on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 125 + - name: findOneAndUpdate + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + - description: "operation succeeds after one socket timeout - bulkWrite on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: bulkWrite + object: *collection + arguments: + requests: + - insertOne: + document: { _id: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation fails after two consecutive socket timeouts - bulkWrite on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 125 + - name: bulkWrite + object: *collection + arguments: + requests: + - insertOne: + document: { _id: 1 } + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + - description: "operation succeeds after one socket timeout - listDatabases on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 125 + - name: listDatabases + object: *client + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - description: "operation fails after two consecutive socket timeouts - listDatabases on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 125 + - name: listDatabases + object: *client + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - description: "operation succeeds after one socket timeout - listDatabaseNames on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 125 + - name: listDatabaseNames + object: *client + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - description: "operation fails after two consecutive socket timeouts - listDatabaseNames on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 125 + - name: listDatabaseNames + object: *client + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + - description: "operation succeeds after one socket timeout - createChangeStream on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *client + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + - description: "operation fails after two consecutive socket timeouts - createChangeStream on client" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *client + arguments: + pipeline: [] + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + - description: "operation succeeds after one socket timeout - aggregate on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: aggregate + object: *database + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - description: "operation fails after two consecutive socket timeouts - aggregate on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: aggregate + object: *database + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - description: "operation succeeds after one socket timeout - listCollections on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 125 + - name: listCollections + object: *database + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - description: "operation fails after two consecutive socket timeouts - listCollections on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 125 + - name: listCollections + object: *database + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - description: "operation succeeds after one socket timeout - listCollectionNames on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 125 + - name: listCollectionNames + object: *database + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - description: "operation fails after two consecutive socket timeouts - listCollectionNames on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 125 + - name: listCollectionNames + object: *database + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + - description: "operation succeeds after one socket timeout - createChangeStream on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *database + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - description: "operation fails after two consecutive socket timeouts - createChangeStream on database" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *database + arguments: + pipeline: [] + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + - description: "operation succeeds after one socket timeout - aggregate on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: aggregate + object: *collection + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - description: "operation fails after two consecutive socket timeouts - aggregate on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: aggregate + object: *collection + arguments: + pipeline: [] + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - description: "operation succeeds after one socket timeout - count on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 125 + - name: count + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - description: "operation fails after two consecutive socket timeouts - count on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 125 + - name: count + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - description: "operation succeeds after one socket timeout - countDocuments on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: countDocuments + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - description: "operation fails after two consecutive socket timeouts - countDocuments on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: countDocuments + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - description: "operation succeeds after one socket timeout - estimatedDocumentCount on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 125 + - name: estimatedDocumentCount + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - description: "operation fails after two consecutive socket timeouts - estimatedDocumentCount on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 125 + - name: estimatedDocumentCount + object: *collection + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + - description: "operation succeeds after one socket timeout - distinct on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["distinct"] + blockConnection: true + blockTimeMS: 125 + - name: distinct + object: *collection + arguments: + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + - description: "operation fails after two consecutive socket timeouts - distinct on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["distinct"] + blockConnection: true + blockTimeMS: 125 + - name: distinct + object: *collection + arguments: + fieldName: x + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + - description: "operation succeeds after one socket timeout - find on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 125 + - name: find + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - description: "operation fails after two consecutive socket timeouts - find on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 125 + - name: find + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - description: "operation succeeds after one socket timeout - findOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 125 + - name: findOne + object: *collection + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - description: "operation fails after two consecutive socket timeouts - findOne on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 125 + - name: findOne + object: *collection + arguments: + filter: {} + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + - description: "operation succeeds after one socket timeout - listIndexes on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listIndexes"] + blockConnection: true + blockTimeMS: 125 + - name: listIndexes + object: *collection + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + - description: "operation fails after two consecutive socket timeouts - listIndexes on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listIndexes"] + blockConnection: true + blockTimeMS: 125 + - name: listIndexes + object: *collection + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + - description: "operation succeeds after one socket timeout - createChangeStream on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *collection + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - description: "operation fails after two consecutive socket timeouts - createChangeStream on collection" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 125 + - name: createChangeStream + object: *collection + arguments: + pipeline: [] + + expectError: + # Network errors are considered client errors by the unified test format spec. + isClientError: true + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + \ No newline at end of file diff --git a/spec/spec_tests/data/client_side_operations_timeout/retryability-timeoutMS.yml b/spec/spec_tests/data/client_side_operations_timeout/retryability-timeoutMS.yml new file mode 100644 index 0000000000..6f47d6c2e4 --- /dev/null +++ b/spec/spec_tests/data/client_side_operations_timeout/retryability-timeoutMS.yml @@ -0,0 +1,2824 @@ +# Tests in this file are generated from retryability-timeoutMS.yml.template. + +description: "timeoutMS behaves correctly for retryable operations" + +schemaVersion: "1.9" + +# failCommand is available on 4.0+ replica sets and 4.2+ sharded clusters. +runOnRequirements: + - minServerVersion: "4.0" + topologies: ["replicaset"] + - minServerVersion: "4.2" + topologies: ["sharded"] + +createEntities: + - client: + id: &failPointClient failPointClient + useMultipleMongoses: false + - client: + id: &client client + uriOptions: + timeoutMS: 100 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + ignoreCommandMonitoringEvents: + - killCursors + - database: + id: &database database + client: *client + databaseName: &databaseName test + - collection: + id: &collection collection + database: *database + collectionName: &collectionName coll + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: [] + +tests: + # For each retryable operation, run three tests: + # + # 1. timeoutMS applies to the whole operation, not to individual attempts - Client timeoutMS=100 and the operation is + # fails with a retryable error after being blocked server-side for 60ms. The operation should fail with a timeout error + # because the second attempt should take it over the 100ms limit. This test only runs on 4.4+ because it uses the + # blockConnection option in failCommand. + # + # 2. operation is retried multiple times if timeoutMS is set to a non-zero value - Client timeoutMS=100 and the + # operation fails with a retryable error twice. Drivers should send the original operation and two retries, the + # second of which should succeed. + # + # 3. operation is retried multiple times if timeoutMS is set to a zero - Override timeoutMS to zero for the operation + # and set a fail point to force a retryable error twice. Drivers should send the original operation and two retries, + # the second of which should succeed. + # + # The fail points in these tests use error code 7 (HostNotFound) because it is a retryable error but does not trigger + # an SDAM state change so we don't lose any time to server rediscovery. The tests also explicitly specify an + # errorLabels array in the fail point to avoid behavioral differences among server types and ensure that the error + # will be considered retryable. + + - description: "timeoutMS applies to whole operation, not individual attempts - insertOne on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: insertOne + object: *collection + arguments: + document: { x: 1 } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - insertOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: insertOne + object: *collection + arguments: + timeoutMS: 1000 + document: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - insertOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: insertOne + object: *collection + arguments: + timeoutMS: 0 + document: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - insertMany on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: insertMany + object: *collection + arguments: + documents: + - { x: 1 } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - insertMany on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: insertMany + object: *collection + arguments: + timeoutMS: 1000 + documents: + - { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - insertMany on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: insertMany + object: *collection + arguments: + timeoutMS: 0 + documents: + - { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - deleteOne on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["delete"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: deleteOne + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - deleteOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["delete"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: deleteOne + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - deleteOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["delete"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: deleteOne + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: delete + databaseName: *databaseName + command: + delete: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - replaceOne on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: replaceOne + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - replaceOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: replaceOne + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - replaceOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: replaceOne + object: *collection + arguments: + timeoutMS: 0 + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - updateOne on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["update"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: updateOne + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - updateOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: updateOne + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - updateOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["update"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: updateOne + object: *collection + arguments: + timeoutMS: 0 + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: update + databaseName: *databaseName + command: + update: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - findOneAndDelete on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: findOneAndDelete + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - findOneAndDelete on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndDelete + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - findOneAndDelete on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndDelete + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - findOneAndReplace on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: findOneAndReplace + object: *collection + arguments: + filter: {} + replacement: { x: 1 } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - findOneAndReplace on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndReplace + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - findOneAndReplace on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndReplace + object: *collection + arguments: + timeoutMS: 0 + filter: {} + replacement: { x: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - findOneAndUpdate on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["findAndModify"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: findOneAndUpdate + object: *collection + arguments: + filter: {} + update: { $set: { x: 1 } } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - findOneAndUpdate on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndUpdate + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - findOneAndUpdate on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["findAndModify"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOneAndUpdate + object: *collection + arguments: + timeoutMS: 0 + filter: {} + update: { $set: { x: 1 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: findAndModify + databaseName: *databaseName + command: + findAndModify: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - bulkWrite on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["insert"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: bulkWrite + object: *collection + arguments: + requests: + - insertOne: + document: { _id: 1 } + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - bulkWrite on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: bulkWrite + object: *collection + arguments: + timeoutMS: 1000 + requests: + - insertOne: + document: { _id: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - bulkWrite on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["insert"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: bulkWrite + object: *collection + arguments: + timeoutMS: 0 + requests: + - insertOne: + document: { _id: 1 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: insert + databaseName: *databaseName + command: + insert: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - listDatabases on client" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: listDatabases + object: *client + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - listDatabases on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listDatabases + object: *client + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - listDatabases on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listDatabases + object: *client + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - listDatabaseNames on client" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["listDatabases"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: listDatabaseNames + object: *client + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - listDatabaseNames on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listDatabaseNames + object: *client + arguments: + timeoutMS: 1000 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - listDatabaseNames on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listDatabases"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listDatabaseNames + object: *client + arguments: + timeoutMS: 0 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listDatabases + databaseName: admin + command: + listDatabases: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - createChangeStream on client" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *client + arguments: + pipeline: [] + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - createChangeStream on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *client + arguments: + timeoutMS: 1000 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - createChangeStream on client" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *client + arguments: + timeoutMS: 0 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: admin + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - aggregate on database" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *database + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - aggregate on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *database + arguments: + timeoutMS: 1000 + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - aggregate on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *database + arguments: + timeoutMS: 0 + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - listCollections on database" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: listCollections + object: *database + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - listCollections on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listCollections + object: *database + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - listCollections on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listCollections + object: *database + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - listCollectionNames on database" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: listCollectionNames + object: *database + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - listCollectionNames on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listCollectionNames + object: *database + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - listCollectionNames on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listCollectionNames + object: *database + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listCollections + databaseName: *databaseName + command: + listCollections: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - createChangeStream on database" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *database + arguments: + pipeline: [] + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - createChangeStream on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *database + arguments: + timeoutMS: 1000 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - createChangeStream on database" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *database + arguments: + timeoutMS: 0 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: 1 + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - aggregate on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *collection + arguments: + pipeline: [] + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - aggregate on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *collection + arguments: + timeoutMS: 1000 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - aggregate on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: aggregate + object: *collection + arguments: + timeoutMS: 0 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - count on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: count + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - count on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: count + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - count on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: count + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - countDocuments on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: countDocuments + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - countDocuments on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: countDocuments + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - countDocuments on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: countDocuments + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - estimatedDocumentCount on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["count"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: estimatedDocumentCount + object: *collection + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - estimatedDocumentCount on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: estimatedDocumentCount + object: *collection + arguments: + timeoutMS: 1000 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - estimatedDocumentCount on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["count"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: estimatedDocumentCount + object: *collection + arguments: + timeoutMS: 0 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: count + databaseName: *databaseName + command: + count: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - distinct on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["distinct"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: distinct + object: *collection + arguments: + fieldName: x + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - distinct on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["distinct"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: distinct + object: *collection + arguments: + timeoutMS: 1000 + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - distinct on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["distinct"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: distinct + object: *collection + arguments: + timeoutMS: 0 + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: distinct + databaseName: *databaseName + command: + distinct: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - find on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: find + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - find on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: find + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - find on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: find + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - findOne on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["find"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: findOne + object: *collection + arguments: + filter: {} + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - findOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOne + object: *collection + arguments: + timeoutMS: 1000 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - findOne on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["find"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: findOne + object: *collection + arguments: + timeoutMS: 0 + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: find + databaseName: *databaseName + command: + find: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - listIndexes on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["listIndexes"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: listIndexes + object: *collection + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - listIndexes on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listIndexes"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listIndexes + object: *collection + arguments: + timeoutMS: 1000 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - listIndexes on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listIndexes"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: listIndexes + object: *collection + arguments: + timeoutMS: 0 + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: listIndexes + databaseName: *databaseName + command: + listIndexes: *collectionName + maxTimeMS: { $$exists: false } + - description: "timeoutMS applies to whole operation, not individual attempts - createChangeStream on collection" + runOnRequirements: + - minServerVersion: "4.4" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 4 } + data: + failCommands: ["aggregate"] + blockConnection: true + blockTimeMS: 60 + errorCode: 7 + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *collection + arguments: + pipeline: [] + + expectError: + isTimeoutError: true + - description: "operation is retried multiple times for non-zero timeoutMS - createChangeStream on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *collection + arguments: + timeoutMS: 1000 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$type: ["int", "long"] } + - description: "operation is retried multiple times if timeoutMS is zero - createChangeStream on collection" + runOnRequirements: + - minServerVersion: "4.3.1" # failCommand errorLabels option + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["aggregate"] + errorCode: 7 + closeConnection: false + errorLabels: ["RetryableWriteError"] + - name: createChangeStream + object: *collection + arguments: + timeoutMS: 0 + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + - commandStartedEvent: + commandName: aggregate + databaseName: *databaseName + command: + aggregate: *collectionName + maxTimeMS: { $$exists: false } + \ No newline at end of file