when connection is lost - identify this on the console

3 files changed, 78 insertions, 14 deletions

diff --git a/scripts/automation/trex_control_plane/client/trex_async_client.py b/scripts/automation/trex_control_plane/client/trex_async_client.py
index 7641a1e3..6793a4ca 100644
--- a/scripts/automation/trex_control_plane/client/trex_async_client.py
+++ b/scripts/automation/trex_control_plane/client/trex_async_client.py
@@ -180,9 +180,18 @@ class CTRexAsyncClient():
 
         self.socket.connect(self.tr)
         self.socket.setsockopt(zmq.SUBSCRIBE, '')
+        self.socket.setsockopt(zmq.RCVTIMEO, 3000)
 
         while self.active:
-            line = self.socket.recv_string();
+            try:
+
+                line = self.socket.recv_string();
+                self.stateless_client.on_async_alive()
+
+            except zmq.Again:
+                self.stateless_client.on_async_dead()
+                continue
+
             msg = json.loads(line)
 
             name = msg['name']
diff --git a/scripts/automation/trex_control_plane/client/trex_stateless_client.py b/scripts/automation/trex_control_plane/client/trex_stateless_client.py
index c5b28f5b..b3476268 100755
--- a/scripts/automation/trex_control_plane/client/trex_stateless_client.py
+++ b/scripts/automation/trex_control_plane/client/trex_stateless_client.py
@@ -425,24 +425,36 @@ class CTRexStatelessClient(object):
 
         self.events = []
 
+
     ################# events handler ######################
-  
+    def add_event_log (self, msg, ev_type, show = False):
+
+        if ev_type == "server":
+            prefix = "[server]"
+        elif ev_type == "local":
+            prefix = "[local]"
+
+        ts = time.time()
+        st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
+        self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold')))
+
+        if show:
+            print format_text("\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))
+
     def handle_async_event (self, type, data):
         # DP stopped
 
-        ev = "[event] - "
-
         show_event = False
 
         # port started
         if (type == 0):
             port_id = int(data['port_id'])
-            ev += "Port {0} has started".format(port_id)
+            ev = "Port {0} has started".format(port_id)
 
         # port stopped
         elif (type == 1):
             port_id = int(data['port_id'])
-            ev += "Port {0} has stopped".format(port_id)
+            ev = "Port {0} has stopped".format(port_id)
 
             # call the handler
             self.async_event_port_stopped(port_id)
@@ -450,14 +462,14 @@ class CTRexStatelessClient(object):
 
         # server stopped
         elif (type == 2):
-            ev += "Server has stopped"
+            ev = "Server has stopped"
             self.async_event_server_stopped()
             show_event = True
 
         # port finished traffic
         elif (type == 3):
             port_id = int(data['port_id'])
-            ev += "Port {0} job done".format(port_id)
+            ev = "Port {0} job done".format(port_id)
 
             # call the handler
             self.async_event_port_stopped(port_id)
@@ -467,12 +479,8 @@ class CTRexStatelessClient(object):
             # unknown event - ignore
             return
 
-        if show_event:
-            print format_text("\n" + ev, 'bold')
 
-        ts = time.time()
-        st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
-        self.events.append("{0} - ".format(st) + format_text(ev, 'bold'))
+        self.add_event_log(ev, 'server', show_event)
 
 
     def async_event_port_stopped (self, port_id):
@@ -580,6 +588,19 @@ class CTRexStatelessClient(object):
         return RC_OK()
 
 
+    def on_async_dead (self):
+        if self.is_connected():
+            msg = 'lost connection to server'
+            self.add_event_log(msg, 'local', True)
+
+            self.disconnect()
+
+    def on_async_alive (self):
+        if not self.is_connected():
+            msg = 'server connection restored'
+            self.add_event_log(msg, 'local', True)
+
+            self.cmd_connect()
 
     ########### cached queries (no server traffic) ###########
 
@@ -1021,7 +1042,7 @@ class CTRexStatelessClient(object):
 
 
         # total has no meaning with percentage - its linear
-        if opts.total and (mult['type'] != 'percentage'):
+        if opts.total and (opts.mult['type'] != 'percentage'):
             # if total was set - divide it between the ports
             opts.mult['max'] = opts.mult['max'] / len(opts.ports)
 
@@ -1202,3 +1223,4 @@ class CTRexStatelessClient(object):
 if __name__ == "__main__":
     pass
 
+
diff --git a/scripts/automation/trex_control_plane/console/trex_console.py b/scripts/automation/trex_control_plane/console/trex_console.py
index 9e44daac..e537c306 100755
--- a/scripts/automation/trex_control_plane/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/console/trex_console.py
@@ -210,6 +210,10 @@ class TRexConsole(TRexGeneralCmd):
     def do_ping (self, line):
         '''Ping the server\n'''
 
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         rc = self.stateless_client.cmd_ping()
         if rc.bad():
             return
@@ -302,6 +306,10 @@ class TRexConsole(TRexGeneralCmd):
     def do_start(self, line):
         '''Start selected traffic in specified port(s) on TRex\n'''
 
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_start_line(line)
 
 
@@ -311,6 +319,11 @@ class TRexConsole(TRexGeneralCmd):
     ############# stop
     def do_stop(self, line):
         '''stops port(s) transmitting traffic\n'''
+
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_stop_line(line)
 
     def help_stop(self):
@@ -319,6 +332,11 @@ class TRexConsole(TRexGeneralCmd):
     ############# update
     def do_update(self, line):
         '''update speed of port(s)currently transmitting traffic\n'''
+
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_update_line(line)
 
     def help_update (self):
@@ -327,11 +345,21 @@ class TRexConsole(TRexGeneralCmd):
     ############# pause
     def do_pause(self, line):
         '''pause port(s) transmitting traffic\n'''
+
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_pause_line(line)
 
     ############# resume
     def do_resume(self, line):
         '''resume port(s) transmitting traffic\n'''
+
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_resume_line(line)
 
    
@@ -339,6 +367,11 @@ class TRexConsole(TRexGeneralCmd):
     ########## reset
     def do_reset (self, line):
         '''force stop all ports\n'''
+
+        if not self.stateless_client.is_connected():
+            print format_text("\nNot connected to server\n", 'bold')
+            return
+
         self.stateless_client.cmd_reset()