Ensure resiliency for route cleanup

db/db_sql.go

@@ -152,6 +152,7 @@ func (s *SqlDB) CleanupRoutes(logger lager.Logger, pruningInterval time.Duration
 		case <-pruningTicker.C:
 			if atomic.CompareAndSwapInt32(&tcpInFlight, 0, 1) {
 				go func() {
+					defer atomic.StoreInt32(&tcpInFlight, 0)
 					var tcpRoutes []models.TcpRouteMapping
 					err := s.FindExpiredRoutes(&tcpRoutes, clock)
 					if err != nil {
@@ -175,12 +176,12 @@ func (s *SqlDB) CleanupRoutes(logger lager.Logger, pruningInterval time.Duration
 					}
 
 					logger.Info("successfully-finished-pruning-tcp-routes", lager.Data{"rowsAffected": rowsAffected})
-					atomic.StoreInt32(&tcpInFlight, 0)
 				}()
 			}
 
 			if atomic.CompareAndSwapInt32(&httpInFlight, 0, 1) {
 				go func() {
+					defer atomic.StoreInt32(&httpInFlight, 0)
 					var httpRoutes []models.Route
 					err := s.FindExpiredRoutes(&httpRoutes, clock)
 					if err != nil {
@@ -204,7 +205,6 @@ func (s *SqlDB) CleanupRoutes(logger lager.Logger, pruningInterval time.Duration
 					}
 
 					logger.Info("successfully-finished-pruning-http-routes", lager.Data{"rowsAffected": rowsAffected})
-					atomic.StoreInt32(&httpInFlight, 0)
 				}()
 			}
 		case <-signals:

db/db_sql_test.go

@@ -1893,6 +1893,39 @@ var _ = Describe("SqlDB", func() {
 					Eventually(logger, 2).Should(gbytes.Say(`failed-to-prune-.*-routes","log_level":2,"data":{"error":"sql: database is closed"}`))
 				})
 			})
+
+			Context("when db throws a temporary error", func() {
+				var (
+					fakeClient *fakes.FakeClient
+					count      int32
+				)
+
+				BeforeEach(func() {
+					fakeClient = &fakes.FakeClient{}
+					sqlDB.Client = fakeClient
+					fakeClient.FindReturns(nil)
+					fakeClient.DeleteStub = func(value interface{}, where ...interface{}) (int64, error) {
+						time.Sleep(500 * time.Millisecond)
+						c := atomic.AddInt32(&count, 1)
+						if c > 5 && c < 10 {
+							return 0, errors.New("temp-error")
+						} else if c >= 10 {
+							return 111, nil
+						} else {
+							return 1, nil
+						}
+					}
+				})
+
+				It("eventually resolves the issue", func() {
+					Eventually(logger, 2).Should(gbytes.Say(`"prune.successfully-finished-pruning-tcp-routes","log_level":1,"data":{"rowsAffected":1}`))
+					Eventually(logger, 2).Should(gbytes.Say(`"prune.successfully-finished-pruning-http-routes","log_level":1,"data":{"rowsAffected":1}`))
+					Eventually(logger, 2).Should(gbytes.Say(`failed-to-prune-tcp-routes","log_level":2,"data":{"error":"temp-error"}`))
+					Eventually(logger, 2).Should(gbytes.Say(`failed-to-prune-http-routes","log_level":2,"data":{"error":"temp-error"}`))
+					Eventually(logger, 2).Should(gbytes.Say(`"prune.successfully-finished-pruning-tcp-routes","log_level":1,"data":{"rowsAffected":111}`))
+					Eventually(logger, 2).Should(gbytes.Say(`"prune.successfully-finished-pruning-http-routes","log_level":1,"data":{"rowsAffected":111}`))
+				})
+			})
 		})
 	}