InterruptedException causing error with properties

[sammefford]

I got the following error with the following code (from #779). The reason I log this against 3.0.8 is because while we don't have DMSDK in 3.0.8, some customers might be writing properties in a multithreaded env. And it appears that writing properties is interacting poorly with OkHttp. What I mean is it appears Okio throws InterruptedException and the com.sun.xml.internal.stream.writers.XMLStreamWriterImpl.flush method doesn't like that so it throws an error.

import com.marklogic.client.DatabaseClient;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.DatabaseClientFactory.BasicAuthContext;
import com.marklogic.client.DatabaseClientFactory.DigestAuthContext;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.datamovement.DataMovementManager;
import com.marklogic.client.datamovement.JobTicket;
import com.marklogic.client.datamovement.WriteBatcher;
import com.icentris.util.CodeTimer;

public class Issue779DMSDK {
  public static void main(String[] args) {
    CodeTimer timer = new CodeTimer();
    timer.start();
    DatabaseClient databaseClient = DatabaseClientFactory.newClient("engrlab-129-192.engrlab.marklogic.com", 8012, new BasicAuthContext("admin", "admin"));
    DataMovementManager dataMovementManager = databaseClient.newDataMovementManager();

    WriteBatcher writeBatcher = dataMovementManager.newWriteBatcher()
      .withBatchSize(100)
      .withThreadCount(64);
    JobTicket jobTicket = dataMovementManager.startJob(writeBatcher);
    DocumentMetadataHandle meta = new DocumentMetadataHandle()
      .withCollections("Issue779DMSDK")
      .withProperty("testProp1", "testValue1");
    for ( int i=0; i < 100000; i++ ) {
      writeBatcher.addAs(i + ".txt", meta, "abc");
    }
    writeBatcher.flushAndWait();
    dataMovementManager.stopJob(jobTicket);
    dataMovementManager.release();
    databaseClient.release();
    timer.stop();
    System.out.println(CodeTimer.getTimingsReport());
  }
}

com.marklogic.client.MarkLogicIOException: Failed to serialize metadata
        at com.marklogic.client.io.DocumentMetadataHandle.sendMetadataImpl(DocumentMetadataHandle.java:801) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.io.DocumentMetadataHandle.write(DocumentMetadataHandle.java:574) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.StreamingOutputImpl.writeTo(StreamingOutputImpl.java:58) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at okhttp3.MultipartBody.writeOrCountBytes(MultipartBody.java:171) ~[okhttp-3.6.0.jar:na]
        at okhttp3.MultipartBody.writeTo(MultipartBody.java:113) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:59) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.6.0.jar:na]
        at com.marklogic.client.impl.HTTPBasicAuthInterceptor.intercept(HTTPBasicAuthInterceptor.java:31) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.6.0.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.6.0.jar:na]
        at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:179) ~[okhttp-3.6.0.jar:na]
        at okhttp3.RealCall.execute(RealCall.java:63) ~[okhttp-3.6.0.jar:na]
        at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:677) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:670) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.doPost(OkHttpServices.java:4006) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.postResource(OkHttpServices.java:3303) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.postBulkDocuments(OkHttpServices.java:3415) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:613) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:605) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl$BatchWriter.run(WriteBatcherImpl.java:1006) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.retry(WriteBatcherImpl.java:479) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.HostAvailabilityListener.processFailure(HostAvailabilityListener.java:157) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.sendThrowableToListeners(WriteBatcherImpl.java:626) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.lambda$newBatchWriteSet$2(WriteBatcherImpl.java:447) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl$BatchWriter.run(WriteBatcherImpl.java:1035) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_71]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_71]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_71]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[na:1.8.0_71]
        at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_71]
Caused by: javax.xml.stream.XMLStreamException: java.io.InterruptedIOException: thread interrupted
        at com.sun.xml.internal.stream.writers.XMLStreamWriterImpl.flush(XMLStreamWriterImpl.java:399) ~[na:1.8.0_71]
        at com.marklogic.client.io.DocumentMetadataHandle.sendMetadataImpl(DocumentMetadataHandle.java:798) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        ... 42 common frames omitted
Caused by: java.io.InterruptedIOException: thread interrupted
        at okio.Timeout.throwIfReached(Timeout.java:146) ~[okio-1.11.0.jar:na]
        at okio.Okio$1.write(Okio.java:75) ~[okio-1.11.0.jar:na]
        at okio.AsyncTimeout$1.write(AsyncTimeout.java:179) ~[okio-1.11.0.jar:na]
        at okio.RealBufferedSink.flush(RealBufferedSink.java:216) ~[okio-1.11.0.jar:na]
        at okhttp3.internal.http1.Http1Codec$ChunkedSink.flush(Http1Codec.java:331) ~[okhttp-3.6.0.jar:na]
        at okio.RealBufferedSink.flush(RealBufferedSink.java:218) ~[okio-1.11.0.jar:na]
        at okio.RealBufferedSink$1.flush(RealBufferedSink.java:199) ~[okio-1.11.0.jar:na]
        at com.sun.xml.internal.stream.writers.UTF8OutputStreamWriter.flush(UTF8OutputStreamWriter.java:138) ~[na:1.8.0_71]
        at com.sun.xml.internal.stream.writers.XMLStreamWriterImpl.flush(XMLStreamWriterImpl.java:397) ~[na:1.8.0_71]
        ... 43 common frames omitted

[sammefford]

@georgeajit do you think you could use ExecutorService to have many threads write docs with properties metadata to reproduce this in 3.0.8 env?

[georgeajit]

Was able to reproduce the issue with 8.0 MarKLogic Server (nightly build) and 3.0-develop for 3.0.8 release.
Here is the stack trace:

java.util.concurrent.ExecutionException: com.marklogic.client.MarkLogicIOException: Failed to serialize metadata
	at java.util.concurrent.FutureTask.report(Unknown Source)
	at java.util.concurrent.FutureTask.get(Unknown Source)
	at com.marklogic.client.functionaltest.TestMultithreading.testMultithreadingMultipleSearch(TestMultithreading.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
	at java.lang.reflect.Method.invoke(Unknown Source)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
	at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
	at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:678)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
Caused by: com.marklogic.client.MarkLogicIOException: Failed to serialize metadata
	at com.marklogic.client.io.DocumentMetadataHandle.sendMetadataImpl(DocumentMetadataHandle.java:688)
	at com.marklogic.client.io.DocumentMetadataHandle.write(DocumentMetadataHandle.java:501)
	at com.marklogic.client.impl.StreamingOutputImpl.writeTo(StreamingOutputImpl.java:58)
	at okhttp3.MultipartBody.writeOrCountBytes(MultipartBody.java:173)
	at okhttp3.MultipartBody.writeTo(MultipartBody.java:114)
	at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:62)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67)
	at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67)
	at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67)
	at com.burgstaller.okhttp.AuthenticationCacheInterceptor.intercept(AuthenticationCacheInterceptor.java:45)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92)
	at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67)
	at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185)
	at okhttp3.RealCall.execute(RealCall.java:69)
	at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:610)
	at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:605)
	at com.marklogic.client.impl.OkHttpServices.doPost(OkHttpServices.java:3829)
	at com.marklogic.client.impl.OkHttpServices.postResource(OkHttpServices.java:3134)
	at com.marklogic.client.impl.OkHttpServices.postBulkDocuments(OkHttpServices.java:3235)
	at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:564)
	at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:557)
	at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:541)
	at com.marklogic.client.functionaltest.ThreadWrite.run(ThreadWrite.java:62)
	at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
	at java.util.concurrent.FutureTask.run(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.lang.Thread.run(Unknown Source)
Caused by: com.ctc.wstx.exc.WstxIOException: thread interrupted
	at com.ctc.wstx.sw.BaseStreamWriter._finishDocument(BaseStreamWriter.java:1448)
	at com.ctc.wstx.sw.BaseStreamWriter.writeEndDocument(BaseStreamWriter.java:556)
	at com.marklogic.client.io.DocumentMetadataHandle.sendMetadataImpl(DocumentMetadataHandle.java:683)
	... 36 more
Caused by: java.io.InterruptedIOException: thread interrupted
	at okio.Timeout.throwIfReached(Timeout.java:145)
	at okio.Okio$1.write(Okio.java:76)
	at okio.AsyncTimeout$1.write(AsyncTimeout.java:180)
	at okio.RealBufferedSink.flush(RealBufferedSink.java:216)
	at okhttp3.internal.http1.Http1Codec$ChunkedSink.flush(Http1Codec.java:331)
	at okio.RealBufferedSink.flush(RealBufferedSink.java:218)
	at okio.RealBufferedSink$1.flush(RealBufferedSink.java:199)
	at com.ctc.wstx.io.UTF8Writer.flush(UTF8Writer.java:103)
	at com.ctc.wstx.sw.BufferingXmlWriter.flush(BufferingXmlWriter.java:242)
	at com.ctc.wstx.sw.BufferingXmlWriter.close(BufferingXmlWriter.java:215)
	at com.ctc.wstx.sw.BaseStreamWriter._finishDocument(BaseStreamWriter.java:1446)
	... 38 more

[georgeajit]

Using the testMultithreadingMultipleSearch() method in Eclipse, above exceptions are thrown.
Note: The intension was to reproduce the same exception as seen in develop and the code below can be enhanced if needed.

/*
 * Copyright 2014-2016 MarkLogic Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.marklogic.client.functionaltest;

import java.io.File;
import java.util.Calendar;

import com.marklogic.client.DatabaseClient;
import com.marklogic.client.document.DocumentWriteSet;
import com.marklogic.client.document.XMLDocumentManager;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.DocumentMetadataHandle.Capability;
import com.marklogic.client.io.StringHandle;

public class ThreadWrite extends Thread{

	String msg;
	DatabaseClient client;
	int nDocCount = 10000;

	public void run()
	{	
		String filename = "flipper.xml";

		File file = new File("src/test/java/com/marklogic/client/functionaltest/data/" + filename);

		XMLDocumentManager docMgr = client.newXMLDocumentManager();
		String[] uris = new String[nDocCount];
			
		    DocumentMetadataHandle mh = setMetadata();
		    DocumentWriteSet writeset = docMgr.newWriteSet();
		    writeset.addDefault(mh);
		    for(int i = 0;i<nDocCount;i++) {
		    	uris[i] = msg + "-" + i+ ".txt";
		    	writeset.add(uris[i], new StringHandle().with("This is so foo"+i));
		    	//System.out.println("Writing " + uris[i] + " to database");

		    	if (msg.contains("WriteThread5") && i == 3500) {
		    		try {
		    			Thread.currentThread().interrupt();
		    		}
		    		catch(Exception e) {
		    			System.out.println("Exception is " + e.getMessage());
		    		}
		    	}
		    }
		   
		    docMgr.write(writeset);
	}

	ThreadWrite(String mg, DatabaseClient dbclient)
	{
		msg=mg;
		client = dbclient;
	}
	
	public DocumentMetadataHandle setMetadata(){
		// create and initialize a handle on the metadata
		DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle();
		metadataHandle.getCollections().addAll("my-collection1","my-collection2");
		metadataHandle.getPermissions().add("app-user", Capability.UPDATE, Capability.READ);
		metadataHandle.getProperties().put("reviewed", true);
		metadataHandle.getProperties().put("myString", "foo");
		metadataHandle.getProperties().put("myInteger", 10);
		metadataHandle.getProperties().put("myDecimal", 34.56678);
		metadataHandle.getProperties().put("myCalendar", Calendar.getInstance().get(Calendar.YEAR));
		metadataHandle.setQuality(23);
		return	metadataHandle;
	}
}

and

/*
 * Copyright 2014-2016 MarkLogic Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.marklogic.client.functionaltest;

import static org.junit.Assert.*;

import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import com.marklogic.client.DatabaseClient;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.DatabaseClientFactory.Authentication;
import com.marklogic.client.document.TextDocumentManager;

import org.junit.*;
public class TestMultithreading extends BasicJavaClientREST {

	private static String dbName = "TestMultithreadingDB";
	private static String [] fNames = {"TestMultithreadingDBDB-1"};
	private static String restServerName = "REST-Java-Client-API-Server";
	private static int restPort=8011;

	@BeforeClass
	public static void setUp() throws Exception 
	{
		System.out.println("In setup");
		setupJavaRESTServer(dbName, fNames[0], restServerName, restPort);
	}

	@After
	public  void testCleanUp() throws Exception
	{
		clearDB(restPort);
		System.out.println("Running clear script");
	}

	@Test
	public void testMultithreading() throws InterruptedException
	{
		ThreadClass dt1 = new ThreadClass("Thread A");
		ThreadClass dt2 = new ThreadClass("Thread B");

		dt1.start(); // this will start thread of object 1
		dt2.start(); // this will start thread of object 2
		dt2.join();

		DatabaseClient client = DatabaseClientFactory.newClient("localhost", 8011, "rest-reader", "x", Authentication.DIGEST);
		TextDocumentManager docMgr = client.newTextDocumentManager();

		for (int i = 1; i <= 5; i++)
		{
			String expectedUri = "/multithread-content-A/filename" + i + ".txt";
			String docUri = docMgr.exists("/multithread-content-A/filename" + i + ".txt").getUri();
			assertEquals("URI is not found", expectedUri, docUri);
		}

		for (int i = 1; i <= 5; i++)
		{
			String expectedUri = "/multithread-content-B/filename" + i + ".txt";
			String docUri = docMgr.exists("/multithread-content-B/filename" + i + ".txt").getUri();
			assertEquals("URI is not found", expectedUri, docUri);
		}

		// release client
		client.release();
	}

	@Test
	public void testMultithreadingMultipleSearch() throws InterruptedException, ExecutionException
	{
		System.out.println("testMultithreadingMultipleSearch");
		
		DatabaseClient client = DatabaseClientFactory.newClient("localhost", 8011, "rest-writer", "x", Authentication.DIGEST);
		ThreadWrite tw1 = new ThreadWrite("WriteThread1", client);
		
		ThreadWrite tw2 = new ThreadWrite("WriteThread2", client);
		
		ThreadWrite tw3 = new ThreadWrite("WriteThread3", client);
		
		ThreadWrite tw4 = new ThreadWrite("WriteThread4", client);
		
		ThreadWrite tw5 = new ThreadWrite("WriteThread5", client);
		
		ExecutorService executor = Executors.newFixedThreadPool(5);
		Future<String> ft1 = (Future<String>) executor.submit(tw1);
		Future<String> ft2 = (Future<String>)executor.submit(tw2);
		Future<String> ft3 = (Future<String>)executor.submit(tw3);
		Future<String> ft4 = (Future<String>)executor.submit(tw4);
		Future<String> ft5 = (Future<String>)executor.submit(tw5);
		
		executor.awaitTermination(1, TimeUnit.MINUTES);
		System.out.println("Future Task 1 is " + ft1.get());
		System.out.println("Future Task 2 is " + ft2.get());
		System.out.println("Future Task 3 is " + ft3.get());
		System.out.println("Future Task 4 is " + ft4.get());
		System.out.println("Future Task 5 is " + ft5.get());
		
		client.release();
	}
	
	@AfterClass	
	public static void tearDown() throws Exception
	{
		System.out.println("In tear down");
		tearDownJavaRESTServer(dbName, fNames, restServerName);
	}
}

[georgeajit]

@sammefford Until further notice, I consider this to be sufficient for you. Please let me know. I ran the test a couple of times. Every time I got the same exception.

[georgeajit]

Without the interrupt, I am not able to to reproduce it. Will keep trying with more threads and document counts.

[georgeajit]

Plan to test writing docs in one thread and updating meta data in another.

[georgeajit]

Tried out the following cases multiple times. None of them reproduce the issue.

1. Write docs in one thread and then update in another; both run parallel. Test fails due to absence of docs during property update.

2. Increased the document count. Write docs and properties in same thread. Test runs fine. Tried with multiple counts and various thread counts.

3. Run the code, posted in the first comment against MarkLogic Server 9.0 and develop branch. I do not get the reported exception.

[sammefford]

Looks like this is related to #646 , since I was mistakenly using one DocumentMetadataHandle across many threads (via DMSDK). I'm seeing now that I was seeing several other errors because of that. When I move DocumentMetadataHandle inside the for loop so it's no longer a shared instance, the errors disappear.

13:24:23.801 [pool-1-thread-61] ERROR c.m.c.d.HostAvailabilityListener - Exception during retry
com.marklogic.client.MarkLogicIOException: javax.xml.stream.XMLStreamException: Attribute not associated with any element
        at com.marklogic.client.io.DocumentMetadataHandle$ValueSerializer.process(DocumentMetadataHandle.java:919) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.ValueConverter.convertFromJava(ValueConverter.java:270) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.ValueConverter.convertFromJava(ValueConverter.java:87) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.io.DocumentMetadataHandle.sendPropertiesImpl(DocumentMetadataHandle.java:888) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.io.DocumentMetadataHandle.sendMetadataImpl(DocumentMetadataHandle.java:791) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.io.DocumentMetadataHandle.write(DocumentMetadataHandle.java:574) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.StreamingOutputImpl.writeTo(StreamingOutputImpl.java:58) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at okhttp3.MultipartBody.writeOrCountBytes(MultipartBody.java:173) ~[okhttp-3.8.1.jar:na]
        at okhttp3.MultipartBody.writeTo(MultipartBody.java:114) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:62) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
        at com.marklogic.client.impl.HTTPBasicAuthInterceptor.intercept(HTTPBasicAuthInterceptor.java:31) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
        at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
        at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) ~[okhttp-3.8.1.jar:na]
        at okhttp3.RealCall.execute(RealCall.java:69) ~[okhttp-3.8.1.jar:na]
        at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:673) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:668) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.doPost(OkHttpServices.java:3991) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.postResource(OkHttpServices.java:3292) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.OkHttpServices.postBulkDocuments(OkHttpServices.java:3400) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:613) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:605) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl$BatchWriter.run(WriteBatcherImpl.java:1000) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.retry(WriteBatcherImpl.java:477) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.HostAvailabilityListener.processFailure(HostAvailabilityListener.java:157) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.sendThrowableToListeners(WriteBatcherImpl.java:624) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.lambda$newBatchWriteSet$2(WriteBatcherImpl.java:445) [marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl$BatchWriter.run(WriteBatcherImpl.java:1026) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_71]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_71]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_71]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[na:1.8.0_71]
        at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_71]
Caused by: javax.xml.stream.XMLStreamException: Attribute not associated with any element
        at com.sun.xml.internal.stream.writers.XMLStreamWriterImpl.writeAttribute(XMLStreamWriterImpl.java:610) ~[na:1.8.0_71]
        at com.marklogic.client.io.DocumentMetadataHandle$ValueSerializer.process(DocumentMetadataHandle.java:915) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
        ... 46 common frames omitted

java.lang.IllegalStateException: This instance has been stopped
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.requireNotStopped(WriteBatcherImpl.java:348)
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.add(WriteBatcherImpl.java:284)
        at com.marklogic.client.datamovement.impl.WriteBatcherImpl.addAs(WriteBatcherImpl.java:332)
        at Issue779DMSDK.main(Issue779DMSDK.java:35)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:497)
        at org.codehaus.mojo.exec.ExecJavaMojo$1.run(ExecJavaMojo.java:282)
        at java.lang.Thread.run(Thread.java:745)

13:24:23.813 [pool-1-thread-31] WARN  c.m.c.d.impl.WriteBatcherImpl - Error writing batch: com.marklogic.client.MarkLogicIOException: Failed to serialize metadata

13:24:23.805 [pool-1-thread-61] WARN  c.m.c.d.impl.WriteBatcherImpl - Error writing batch: com.marklogic.client.MarkLogicIOException: javax.xml.stream.XMLStreamException: java.net.SocketException: Broken pipe

[sammefford]

I'm going to call this a red herring. Since Ajit can't reproduce the bug and I can't without misusing DocumentMetadataHandle, I can't say this is really a bug.