Tree: fix error handling when gateway channel is closing

Fix PropagationChannel.ev_close() where gateway channel termination is
handled.
If we get an actual rc > 0, that comes from the gateway command itself
and that means the gateway is defective/misconfigured, in that case, we
mark it as unreachable at the Task level.
In addition, in that case, if we have not launched the remote commands
yet, they are redistributed to other available gateways.

rc=None is now handled as a normal termination of the propagation
channel and the corresponding gateway is not marked as unreachable
anymore.

Fixes #566.

Author: thiell

lib/ClusterShell/Propagation.py

@@ -394,24 +394,32 @@ def recv_ctl(self, msg):
 
     def ev_hup(self, worker, node, rc):
         """Channel command is closing"""
+        self.logger.debug("ev_hup gateway=%s %s", str(worker.nodes), self)
         self._rc = rc
 
     def ev_close(self, worker, timedout):
         """Channel is closing"""
         # do not use worker buffer or rc accessors here as we doesn't use
         # common stream names
         gateway = str(worker.nodes)
-        self.logger.debug("ev_close gateway=%s %s", gateway, self)
-        self.logger.debug("ev_close rc=%s", self._rc) # may be None
-
-        # NOTE: self._rc may be None if the communication channel has aborted
-        if self._rc != 0:
-            self.logger.debug("error on gateway %s (setup=%s)", gateway,
-                              self.setup)
+        self.logger.debug("ev_close gateway=%s rc=%s %s", gateway, self._rc,
+                          self)
+
+        # NOTE: self._rc is set None when _we_ close the channel (abort)
+        if self._rc is None and not self.setup:
+            # aborting before the channel is setup is worth a warning
+            self.logger.warning("ev_close: rc=%s with channel not setup",
+                                self._rc)
+
+        if self._rc is not None and self._rc != 0:
+            # handle gateway channel error
+            self.logger.debug("error on gateway %s (rc=%s, setup=%s)", gateway,
+                              self._rc, self.setup)
             self.task.router.mark_unreachable(gateway)
             self.logger.debug("gateway %s now set as unreachable", gateway)
 
             if not self.setup:
-                # channel was not set up: we can safely repropagate commands
+                # channel was not set up: we can safely redistribute commands
+                self.logger.debug("channel was not set up: redistributing...")
                 for mw in set(self.task.gateways[gateway][1]):
                     mw._relaunch(gateway)

lib/ClusterShell/Worker/Tree.py

@@ -430,8 +430,8 @@ def _on_remote_node_msgline(self, node, msg, sname, gateway):
     def _on_remote_node_close(self, node, rc, gateway):
         """remote node closing with return code"""
         DistantWorker._on_node_close(self, node, rc)
-        self.logger.debug("_on_remote_node_close %s %s via gw %s", node,
-                          self._close_count, gateway)
+        self.logger.debug("_on_remote_node_close %s %s via gw %s rc=%s", node,
+                          self._close_count, gateway, rc)
 
         # finalize rcopy: extract tar data
         if self.source and self.reverse: