retest driver connections during driver discovery to make sure we can recover from hung requests to voldrivers [#154852943](https://www.pivotaltracker.com/story/show/154852943)

Author: Julian Hjortshoj

vollocal/docker_driver_discoverer.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-
+	"errors"
 	"context"
 
 	"code.cloudfoundry.org/lager"
@@ -135,6 +135,16 @@ func (r *dockerDriverDiscoverer) insertIfAliveAndNotFound(logger lager.Logger, e
 					logger.Info("existing-driver-mismatch", lager.Data{"specName": specName, "address": driverSpec.Address, "tls": driverSpec.TLSConfig})
 					plugin = nil
 				}
+				if plugin != nil {
+					dockerPlugin := plugin.(*driverhttp.DockerDriverPlugin)
+					dockerDriver := dockerPlugin.DockerDriver.(voldriver.Driver)
+					env := driverhttp.NewHttpDriverEnv(logger, context.Background())
+					resp := dockerDriver.Activate(env)
+					if resp.Err != "" {
+						logger.Error("existing-driver-unreachable", errors.New(resp.Err), lager.Data{"specName": specName, "address": driverSpec.Address, "tls": driverSpec.TLSConfig})
+						plugin = nil
+					}
+				}
 			}
 
 			if plugin == nil {

vollocal/docker_driver_discoverer_test.go

@@ -109,7 +109,17 @@ var _ = Describe("Docker Driver Discoverer", func() {
 					// Expect SetDrivers not to be called
 					Expect(len(drivers)).To(Equal(1))
 					Expect(fakeDriverFactory.DockerDriverCallCount()).To(Equal(1))
-					Expect(fakeDriver.ActivateCallCount()).To(Equal(1))
+					Expect(fakeDriver.ActivateCallCount()).To(Equal(2))
+				})
+				Context("when the existing driver connection is broken", func(){
+					BeforeEach(func(){
+						fakeDriver.ActivateReturnsOnCall(1, voldriver.ActivateResponse{Err: "badness"})
+					})
+					It("should replace the driver in the registry", func() {
+						Expect(len(drivers)).To(Equal(1))
+						Expect(fakeDriverFactory.DockerDriverCallCount()).To(Equal(2))
+						Expect(fakeDriver.ActivateCallCount()).To(Equal(3))
+					})
 				})
 			})