TUI v2.0 - now no flickering, state machine for lost of connectivity

and TUI can be started in xterm using tui -x

8 files changed, 181 insertions, 42 deletions

diff --git a/scripts/automation/trex_control_plane/client/trex_async_client.py b/scripts/automation/trex_control_plane/client/trex_async_client.py
index 8fdf7c9b..00304886 100644
--- a/scripts/automation/trex_control_plane/client/trex_async_client.py
+++ b/scripts/automation/trex_control_plane/client/trex_async_client.py
@@ -152,16 +152,19 @@ class CTRexAsyncStatsManager():
 
 
 class CTRexAsyncClient():
-    def __init__ (self, server, port, stateless_client):
+    def __init__ (self, server, port, stateless_client, prn_func = None):
 
         self.port = port
         self.server = server
         self.stateless_client = stateless_client
+        self.prn_func = prn_func
 
         self.raw_snapshot = {}
 
         self.stats = CTRexAsyncStatsManager()
 
+        self.last_data_recv_ts = 0
+
         self.connected = False
  
     # connects the async channel
@@ -171,7 +174,13 @@ class CTRexAsyncClient():
             self.disconnect()
 
         self.tr = "tcp://{0}:{1}".format(self.server, self.port)
-        print "\nConnecting To ZMQ Publisher On {0}".format(self.tr)
+
+        msg = "\nConnecting To ZMQ Publisher On {0}".format(self.tr)
+
+        if self.prn_func:
+            self.prn_func(msg)
+        else:
+            print msg
 
         #  Socket to talk to server
         self.context = zmq.Context()
@@ -180,7 +189,6 @@ class CTRexAsyncClient():
 
         # before running the thread - mark as active
         self.active = True
-        self.alive = False
         self.t = threading.Thread(target = self._run)
 
         # kill this thread on exit and don't add it to the join list
@@ -192,7 +200,7 @@ class CTRexAsyncClient():
 
         # wait for data streaming from the server
         timeout = time.time() + 5
-        while not self.alive:
+        while not self.is_alive():
             time.sleep(0.01)
             if time.time() > timeout:
                 self.disconnect()
@@ -219,35 +227,38 @@ class CTRexAsyncClient():
     # thread function
     def _run (self):
 
-        # no data yet...
-        self.alive = False
 
         # socket must be created on the same thread 
         self.socket.connect(self.tr)
         self.socket.setsockopt(zmq.SUBSCRIBE, '')
         self.socket.setsockopt(zmq.RCVTIMEO, 5000)
 
+        got_data = False
+
         while self.active:
             try:
 
                 line = self.socket.recv_string()
+                self.last_data_recv_ts = time.time()
 
-                if not self.alive:
+                # signal once
+                if not got_data:
                     self.stateless_client.on_async_alive()
-                    self.alive = True
+                    got_data = True
+                
 
             # got a timeout - mark as not alive and retry
             except zmq.Again:
 
-                if self.alive:
+                # signal once
+                if got_data:
                     self.stateless_client.on_async_dead()
-                    self.alive = False
+                    got_data = False
 
                 continue
 
             except zmq.ContextTerminated:
                 # outside thread signaled us to exit
-                self.alive = False
                 break
 
             msg = json.loads(line)
@@ -264,6 +275,13 @@ class CTRexAsyncClient():
         self.socket.close(linger = 0)
 
 
+    # did we get info for the last 3 seconds ?
+    def is_alive (self):
+        if self.last_data_recv_ts == None:
+            return False
+
+        return ( (time.time() - self.last_data_recv_ts) < 3 )
+
     def get_stats (self):
         return self.stats
 
diff --git a/scripts/automation/trex_control_plane/client/trex_port.py b/scripts/automation/trex_control_plane/client/trex_port.py
index 5c5702dd..4f82e86a 100644
--- a/scripts/automation/trex_control_plane/client/trex_port.py
+++ b/scripts/automation/trex_control_plane/client/trex_port.py
@@ -387,6 +387,9 @@ class Port(object):
     def clear_stats(self):
         return self.port_stats.clear_stats()
 
+    def invalidate_stats(self):
+        return self.port_stats.invalidate()
+
 
     ################# events handler ######################
     def async_event_port_stopped (self):
diff --git a/scripts/automation/trex_control_plane/client/trex_stateless_client.py b/scripts/automation/trex_control_plane/client/trex_stateless_client.py
index a2b1f6d9..899805cf 100755
--- a/scripts/automation/trex_control_plane/client/trex_stateless_client.py
+++ b/scripts/automation/trex_control_plane/client/trex_stateless_client.py
@@ -55,7 +55,7 @@ class CTRexStatelessClient(object):
 
         self.user = username
 
-        self.comm_link = CTRexStatelessClient.CCommLink(server, sync_port, virtual)
+        self.comm_link = CTRexStatelessClient.CCommLink(server, sync_port, virtual, self.prn_func)
 
         # default verbose level
         self.verbose = self.VERBOSE_REGULAR
@@ -68,7 +68,7 @@ class CTRexStatelessClient(object):
         self.server_version = {}
         self.__err_log = None
 
-        self.async_client = CTRexAsyncClient(server, async_port, self)
+        self.async_client = CTRexAsyncClient(server, async_port, self, self.prn_func)
 
         self.streams_db = CStreamsDB()
         self.global_stats = trex_stats.CGlobalStats(self._connection_info,
@@ -105,8 +105,8 @@ class CTRexStatelessClient(object):
         st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
         self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold')))
 
-        if show and self.check_verbose(self.VERBOSE_REGULAR):
-            print format_text("\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))
+        if show:
+            self.prn_func(format_text("\n{:^8} - {:}".format(prefix, format_text(msg, 'bold'))))
   
 
     def handle_async_stats_update(self, dump_data):
@@ -473,6 +473,10 @@ class CTRexStatelessClient(object):
     def get_verbose (self):
         return self.verbose
 
+    def prn_func (self, msg, level = VERBOSE_REGULAR):
+        if self.check_verbose(level):
+            print msg
+
     ############# server actions ################
 
     # ping server
@@ -754,6 +758,14 @@ class CTRexStatelessClient(object):
         return RC_OK()
 
 
+    def cmd_invalidate (self, port_id_list):
+        for port_id in port_id_list:
+            self.ports[port_id].invalidate_stats()
+
+        self.global_stats.invalidate()
+
+        return RC_OK()
+
     # pause cmd
     def cmd_pause (self, port_id_list):
 
@@ -1146,13 +1158,13 @@ class CTRexStatelessClient(object):
     # ------ private classes ------ #
     class CCommLink(object):
         """describes the connectivity of the stateless client method"""
-        def __init__(self, server="localhost", port=5050, virtual=False):
+        def __init__(self, server="localhost", port=5050, virtual=False, prn_func = None):
             super(CTRexStatelessClient.CCommLink, self).__init__()
             self.virtual = virtual
             self.server = server
             self.port = port
             self.verbose = False
-            self.rpc_link = JsonRpcClient(self.server, self.port)
+            self.rpc_link = JsonRpcClient(self.server, self.port, prn_func)
 
         @property
         def is_connected(self):
diff --git a/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py b/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py
index 3de0bb5f..ce98fbc6 100755
--- a/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py
+++ b/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py
@@ -47,7 +47,7 @@ class BatchMessage(object):
 # JSON RPC v2.0 client
 class JsonRpcClient(object):
 
-    def __init__ (self, default_server, default_port):
+    def __init__ (self, default_server, default_port, prn_func = None):
         self.verbose = False
         self.connected = False
 
@@ -56,6 +56,8 @@ class JsonRpcClient(object):
         self.server = default_server
         self.id_gen = general_utils.random_id_gen()
 
+        self.prn_func = prn_func
+
     def get_connection_details (self):
         rc = {}
         rc['server'] = self.server
@@ -201,7 +203,8 @@ class JsonRpcClient(object):
         else:
             return False, "Not connected to server"
 
-    def connect(self, server=None, port=None):
+
+    def connect(self, server = None, port = None, prn_func = None):
         if self.connected:
             self.disconnect()
 
@@ -213,7 +216,11 @@ class JsonRpcClient(object):
         #  Socket to talk to server
         self.transport = "tcp://{0}:{1}".format(self.server, self.port)
 
-        print "\nConnecting To RPC Server On {0}".format(self.transport)
+        msg = "\nConnecting To RPC Server On {0}".format(self.transport)
+        if self.prn_func:
+            self.prn_func(msg)
+        else:
+            print msg
 
         self.socket = self.context.socket(zmq.REQ)
         try:
diff --git a/scripts/automation/trex_control_plane/client_utils/parsing_opts.py b/scripts/automation/trex_control_plane/client_utils/parsing_opts.py
index 7ac9e312..6f9b4c6d 100755
--- a/scripts/automation/trex_control_plane/client_utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/client_utils/parsing_opts.py
@@ -21,12 +21,13 @@ STREAM_FROM_PATH_OR_FILE = 9
 DURATION = 10
 FORCE = 11
 DRY_RUN = 12
-TOTAL = 13
+XTERM = 13
+TOTAL = 14
 
-GLOBAL_STATS = 14
-PORT_STATS = 15
-PORT_STATUS = 16
-STATS_MASK = 17
+GLOBAL_STATS = 50
+PORT_STATS = 51
+PORT_STATUS = 52
+STATS_MASK = 53
 
 # list of ArgumentGroup types
 MUTEX = 1
@@ -210,6 +211,13 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
                                      'default': False,
                                      'help': "Dry run - no traffic will be injected"}),
 
+
+              XTERM: ArgumentPack(['-x', '--xterm'],
+                                  {'action': 'store_true',
+                                   'dest': 'xterm',
+                                   'default': False,
+                                   'help': "Starts TUI in xterm window"}),
+
               GLOBAL_STATS: ArgumentPack(['-g'],
                                          {'action': 'store_true',
                                           'help': "Fetch only global statistics"}),
diff --git a/scripts/automation/trex_control_plane/common/trex_stats.py b/scripts/automation/trex_control_plane/common/trex_stats.py
index 2f6ea38d..20255f41 100755
--- a/scripts/automation/trex_control_plane/common/trex_stats.py
+++ b/scripts/automation/trex_control_plane/common/trex_stats.py
@@ -194,6 +194,9 @@ class CTRexStats(object):
 
     @staticmethod
     def format_num(size, suffix = ""):
+        if type(size) == str:
+            return "N/A"
+
         for unit in ['','K','M','G','T','P']:
             if abs(size) < 1000.0:
                 return "%3.2f %s%s" % (size, unit, suffix)
@@ -219,6 +222,9 @@ class CTRexStats(object):
     def clear_stats(self):
         self.reference_stats = self.latest_stats
 
+    def invalidate (self):
+        self.latest_stats = {}
+
     def get(self, field, format=False, suffix=""):
         if not field in self.latest_stats:
             return "N/A"
diff --git a/scripts/automation/trex_control_plane/console/trex_console.py b/scripts/automation/trex_control_plane/console/trex_console.py
index 0ecfce9c..325ba514 100755
--- a/scripts/automation/trex_control_plane/console/trex_console.py
+++ b/scripts/automation/trex_control_plane/console/trex_console.py
@@ -17,7 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-
+import subprocess
 import cmd
 import json
 import ast
@@ -455,6 +455,19 @@ class TRexConsole(TRexGeneralCmd):
     def do_tui (self, line):
         '''Shows a graphical console\n'''
 
+        parser = parsing_opts.gen_parser(self,
+                                         "tui",
+                                         self.do_tui.__doc__,
+                                         parsing_opts.XTERM)
+
+        opts = parser.parse_args(line.split())
+        if opts is None:
+            return
+
+        if opts.xterm:
+            subprocess.Popen(['xterm', '-geometry', '105x40', '-e', './trex-console', '-t'])
+            return
+
         save_verbose = self.stateless_client.get_verbose()
 
         self.stateless_client.set_verbose(self.stateless_client.VERBOSE_SILENCE)
diff --git a/scripts/automation/trex_control_plane/console/trex_tui.py b/scripts/automation/trex_control_plane/console/trex_tui.py
index 2e6be4a6..3a89097f 100644
--- a/scripts/automation/trex_control_plane/console/trex_tui.py
+++ b/scripts/automation/trex_control_plane/console/trex_tui.py
@@ -8,6 +8,22 @@ from client_utils import text_tables
 from collections import OrderedDict
 import datetime
 
+class SimpleBar(object):
+    def __init__ (self, desc, pattern):
+        self.desc = desc
+        self.pattern = pattern
+        self.pattern_len = len(pattern)
+        self.index = 0
+
+    def show (self):
+        if self.desc:
+            print format_text("{0} {1}".format(self.desc, self.pattern[self.index]), 'bold')
+        else:
+            print format_text("{0}".format(self.pattern[self.index]), 'bold')
+
+        self.index = (self.index + 1) % self.pattern_len
+
+
 # base type of a panel
 class TrexTUIPanel(object):
     def __init__ (self, mng, name):
@@ -224,6 +240,7 @@ class TrexTUILog():
         self.log.append("[{0}] {1}".format(str(datetime.datetime.now().time()), msg))
 
     def show (self, max_lines = 4):
+
         cut = len(self.log) - max_lines
         if cut < 0:
             cut = 0
@@ -261,6 +278,10 @@ class TrexTUIPanelManager():
 
         self.generate_legend()
 
+        self.conn_bar = SimpleBar('status: ', ['|','/','-','\\'])
+        self.dis_bar =  SimpleBar('status: ', ['X', ' '])
+        self.show_log = False
+
 
     def generate_legend (self):
         self.legend = "\n{:<12}".format("browse:")
@@ -280,18 +301,28 @@ class TrexTUIPanelManager():
                 self.legend += "{:}".format(x)
 
 
+    def print_connection_status (self):
+        if self.tui.get_state() == self.tui.STATE_ACTIVE:
+            self.conn_bar.show()
+        else:
+            self.dis_bar.show()
+
     def print_legend (self):
         print format_text(self.legend, 'bold')
 
 
     # on window switch or turn on / off of the TUI we call this
-    def init (self):
+    def init (self, show_log = False):
+        self.show_log = show_log
         self.generate_legend()
 
     def show (self):
         self.main_panel.show()
+        self.print_connection_status()
         self.print_legend()
-        self.log.show()
+
+        if self.show_log:
+            self.log.show()
 
 
     def handle_key (self, ch):
@@ -323,7 +354,7 @@ class TrexTUIPanelManager():
 
     def action_show_dash (self):
         self.main_panel = self.panels['dashboard']
-        self.init()
+        self.init(self.show_log)
         return ""
 
     def action_show_port (self, port_id):
@@ -338,6 +369,11 @@ class TrexTUIPanelManager():
 
 # shows a textual top style window
 class TrexTUI():
+
+    STATE_ACTIVE     = 0
+    STATE_LOST_CONT  = 1
+    STATE_RECONNECT  = 2
+
     def __init__ (self, stateless_client):
         self.stateless_client = stateless_client
 
@@ -349,10 +385,10 @@ class TrexTUI():
         # try to read a single key
         ch = os.read(sys.stdin.fileno(), 1)
         if ch != None and len(ch) > 0:
-            return self.pm.handle_key(ch)
+            return (self.pm.handle_key(ch), True)
 
         else:
-            return True
+            return (True, False)
             
 
     def clear_screen (self):
@@ -360,7 +396,7 @@ class TrexTUI():
 
 
 
-    def show (self):
+    def show (self, show_log = False):
         # init termios
         old_settings = termios.tcgetattr(sys.stdin)
         new_settings = termios.tcgetattr(sys.stdin)
@@ -369,27 +405,63 @@ class TrexTUI():
         new_settings[6][termios.VTIME] = 0 # cc
         termios.tcsetattr(sys.stdin, termios.TCSADRAIN, new_settings)
 
-        self.pm.init()
+        self.pm.init(show_log)
+
+        self.state = self.STATE_ACTIVE
+        self.draw_policer = 0
 
         try:
             while True:
-                self.clear_screen()
-
-                cont = self.handle_key_input()
-                self.pm.show()
-
+                # draw and handle user input
+                cont, force_draw = self.handle_key_input()
+                self.draw_screen(force_draw)
                 if not cont:
                     break
-
                 time.sleep(0.1)
 
+                # regular state
+                if self.state == self.STATE_ACTIVE:
+                    # if no connectivity - move to lost connecitivty
+                    if not self.stateless_client.async_client.is_alive():
+                        self.stateless_client.cmd_invalidate(self.pm.ports)
+                        self.state = self.STATE_LOST_CONT
+
+
+                # lost connectivity
+                elif self.state == self.STATE_LOST_CONT:
+                    # got it back
+                    if self.stateless_client.async_client.is_alive():
+                        # move to state reconnect
+                        self.state = self.STATE_RECONNECT
+
+
+                # restored connectivity - try to reconnect
+                elif self.state == self.STATE_RECONNECT:
+
+                    rc = self.stateless_client.connect("RO")
+                    if rc.good():
+                        self.state = self.STATE_ACTIVE
+                    else:
+                        # maybe we lost it again
+                        self.state = self.STATE_LOST_CONT
+
+
         finally:
             # restore
             termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
 
         print ""
 
-    # key actions
-    def action_quit (self):
-        return False
 
+    # draw once
+    def draw_screen (self, force_draw = False):
+
+        if (self.draw_policer >= 5) or (force_draw):
+            self.clear_screen()
+            self.pm.show()
+            self.draw_policer = 0
+        else:
+            self.draw_policer += 1 
+
+    def get_state (self):
+        return self.state