All documents not returned after failover #844
Description
I am seeing this issue for the first time now in Windows cluster. In the following test run on a 3 node cluster, I stop node 3 , after 5 seconds , stop node 2 and start node 3. Intermittently, I see 13891 docs returned . The server and client logs are attached.
TEST-com.marklogic.client.datamovement.functionaltests.QBFailover_1.txt
serverLog.txt
@Test
public void testStopTwoNodes() throws Exception {
System.out.println(Thread.currentThread().getStackTrace()[1].getMethodName());
try {
final AtomicInteger success = new AtomicInteger(0);
AtomicBoolean isNode3Running = new AtomicBoolean(true);
AtomicBoolean isNode2Running = new AtomicBoolean(true);
QueryBatcher batcher = dmManager.newQueryBatcher(new StructuredQueryBuilder().collection("XmlTransform"))
.withBatchSize(5).withThreadCount(6);
HostAvailabilityListener.getInstance(batcher).withSuspendTimeForHostUnavailable(Duration.ofSeconds(15))
.withMinHosts(1);
NoResponseListener.getInstance(batcher).withSuspendTimeForHostUnavailable(Duration.ofSeconds(15))
.withMinHosts(1);
batcher.onUrisReady((batch) -> {
success.addAndGet(batch.getItems().length);
}).onQueryFailure(queryException -> {
queryException.printStackTrace();
});
ticket = dmManager.startJob(batcher);
while (!batcher.isStopped()) {
if (isNode3Running.get() && dmManager.getJobReport(ticket).getSuccessEventsCount() > 0) {
isNode3Running.set(false);
serverStartStop(hostNames[hostNames.length - 1], "stop");
}
if (isNode2Running.get() && dmManager.getJobReport(ticket).getSuccessEventsCount() > 50) {
isNode2Running.set(false);
serverStartStop(hostNames[hostNames.length - 2], "stop");
Thread.currentThread().sleep(5000L);
serverStartStop(hostNames[hostNames.length - 1], "start");
}
}
batcher.awaitCompletion();
dmManager.stopJob(ticket);
System.out.println("Success " + success.intValue());
assertEquals("document count", 20000, success.intValue());
} catch (Exception e) {
e.printStackTrace();
}
}