From 995267db77f5554d5228697b8b2a862b51859fe6 Mon Sep 17 00:00:00 2001 From: imarom Date: Mon, 8 Feb 2016 06:08:14 -0500 Subject: first refactor --- .../trex_control_plane/client/trex_async_client.py | 330 ---- .../trex_control_plane/client/trex_port.py | 512 ----- .../client/trex_stateless_client.py | 2035 -------------------- .../client/trex_stateless_sim.py | 430 ----- .../client_utils/jsonrpc_client.py | 244 --- .../client_utils/packet_builder.py | 1209 ------------ .../client_utils/packet_builder_interface.py | 43 - .../client_utils/scapy_packet_builder.py | 748 ------- .../trex_control_plane/client_utils/text_tables.py | 4 +- .../trex_control_plane/client_utils/yaml_utils.py | 1 - .../trex_control_plane/common/rpc_defaults.yaml | 124 -- .../trex_control_plane/common/trex_stats.py | 578 ------ .../common/trex_stl_exceptions.py | 53 - .../trex_control_plane/common/trex_streams.py | 526 ----- .../trex_control_plane/common/trex_types.py | 95 - .../trex_control_plane/console/__init__.py | 0 .../trex_control_plane/console/trex_console.py | 16 +- .../trex_control_plane/console/trex_tui.py | 14 +- .../automation/trex_control_plane/stl/__init__.py | 0 scripts/automation/trex_control_plane/stl/api.py | 31 + .../trex_control_plane/stl/rpc_defaults.yaml | 124 ++ .../stl/trex_stl_async_client.py | 322 ++++ .../trex_control_plane/stl/trex_stl_client.py | 2020 +++++++++++++++++++ .../trex_control_plane/stl/trex_stl_exceptions.py | 54 + .../stl/trex_stl_jsonrpc_client.py | 243 +++ .../stl/trex_stl_packet_builder_interface.py | 43 + .../stl/trex_stl_packet_builder_scapy.py | 747 +++++++ .../trex_control_plane/stl/trex_stl_port.py | 512 +++++ .../trex_control_plane/stl/trex_stl_sim.py | 396 ++++ .../trex_control_plane/stl/trex_stl_stats.py | 581 ++++++ .../trex_control_plane/stl/trex_stl_std.py | 67 + .../trex_control_plane/stl/trex_stl_streams.py | 230 +++ .../trex_control_plane/stl/trex_stl_types.py | 95 + 33 files changed, 5483 insertions(+), 6944 deletions(-) delete mode 100644 scripts/automation/trex_control_plane/client/trex_async_client.py delete mode 100644 scripts/automation/trex_control_plane/client/trex_port.py delete mode 100755 scripts/automation/trex_control_plane/client/trex_stateless_client.py delete mode 100644 scripts/automation/trex_control_plane/client/trex_stateless_sim.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py delete mode 100755 scripts/automation/trex_control_plane/client_utils/packet_builder.py delete mode 100644 scripts/automation/trex_control_plane/client_utils/packet_builder_interface.py delete mode 100644 scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py delete mode 100755 scripts/automation/trex_control_plane/common/rpc_defaults.yaml delete mode 100755 scripts/automation/trex_control_plane/common/trex_stats.py delete mode 100644 scripts/automation/trex_control_plane/common/trex_stl_exceptions.py delete mode 100755 scripts/automation/trex_control_plane/common/trex_streams.py delete mode 100644 scripts/automation/trex_control_plane/common/trex_types.py create mode 100644 scripts/automation/trex_control_plane/console/__init__.py create mode 100644 scripts/automation/trex_control_plane/stl/__init__.py create mode 100644 scripts/automation/trex_control_plane/stl/api.py create mode 100644 scripts/automation/trex_control_plane/stl/rpc_defaults.yaml create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_async_client.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_client.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_port.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_sim.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_stats.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_std.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_streams.py create mode 100644 scripts/automation/trex_control_plane/stl/trex_stl_types.py (limited to 'scripts/automation/trex_control_plane') diff --git a/scripts/automation/trex_control_plane/client/trex_async_client.py b/scripts/automation/trex_control_plane/client/trex_async_client.py deleted file mode 100644 index ef4c48f9..00000000 --- a/scripts/automation/trex_control_plane/client/trex_async_client.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/router/bin/python - -try: - # support import for Python 2 - import outer_packages -except ImportError: - # support import for Python 3 - import client.outer_packages -from client_utils.jsonrpc_client import JsonRpcClient, BatchMessage - -from common.text_opts import * - -import json -import threading -import time -import datetime -import zmq -import re -import random - -from common.trex_stats import * -from common.trex_streams import * -from common.trex_types import * - -# basic async stats class -class CTRexAsyncStats(object): - def __init__ (self): - self.ref_point = None - self.current = {} - self.last_update_ts = datetime.datetime.now() - - def update (self, snapshot): - - #update - self.last_update_ts = datetime.datetime.now() - - self.current = snapshot - - if self.ref_point == None: - self.ref_point = self.current - - def clear(self): - self.ref_point = self.current - - - def get(self, field, format=False, suffix=""): - - if not field in self.current: - return "N/A" - - if not format: - return self.current[field] - else: - return format_num(self.current[field], suffix) - - def get_rel (self, field, format=False, suffix=""): - if not field in self.current: - return "N/A" - - if not format: - return (self.current[field] - self.ref_point[field]) - else: - return format_num(self.current[field] - self.ref_point[field], suffix) - - - # return true if new data has arrived in the past 2 seconds - def is_online (self): - delta_ms = (datetime.datetime.now() - self.last_update_ts).total_seconds() * 1000 - return (delta_ms < 2000) - -# describes the general stats provided by TRex -class CTRexAsyncStatsGeneral(CTRexAsyncStats): - def __init__ (self): - super(CTRexAsyncStatsGeneral, self).__init__() - - -# per port stats -class CTRexAsyncStatsPort(CTRexAsyncStats): - def __init__ (self): - super(CTRexAsyncStatsPort, self).__init__() - - def get_stream_stats (self, stream_id): - return None - -# stats manager -class CTRexAsyncStatsManager(): - def __init__ (self): - - self.general_stats = CTRexAsyncStatsGeneral() - self.port_stats = {} - - - def get_general_stats(self): - return self.general_stats - - def get_port_stats (self, port_id): - - if not str(port_id) in self.port_stats: - return None - - return self.port_stats[str(port_id)] - - - def update(self, data): - self.__handle_snapshot(data) - - def __handle_snapshot(self, snapshot): - - general_stats = {} - port_stats = {} - - # filter the values per port and general - for key, value in snapshot.iteritems(): - - # match a pattern of ports - m = re.search('(.*)\-([0-8])', key) - if m: - - port_id = m.group(2) - field_name = m.group(1) - - if not port_id in port_stats: - port_stats[port_id] = {} - - port_stats[port_id][field_name] = value - - else: - # no port match - general stats - general_stats[key] = value - - # update the general object with the snapshot - self.general_stats.update(general_stats) - - # update all ports - for port_id, data in port_stats.iteritems(): - - if not port_id in self.port_stats: - self.port_stats[port_id] = CTRexAsyncStatsPort() - - self.port_stats[port_id].update(data) - - - - - -class CTRexAsyncClient(): - def __init__ (self, server, port, stateless_client): - - self.port = port - self.server = server - - self.stateless_client = stateless_client - - self.event_handler = stateless_client.event_handler - self.logger = self.stateless_client.logger - - self.raw_snapshot = {} - - self.stats = CTRexAsyncStatsManager() - - self.last_data_recv_ts = 0 - self.async_barrier = None - - self.connected = False - - # connects the async channel - def connect (self): - - if self.connected: - self.disconnect() - - self.tr = "tcp://{0}:{1}".format(self.server, self.port) - - # Socket to talk to server - self.context = zmq.Context() - self.socket = self.context.socket(zmq.SUB) - - - # before running the thread - mark as active - self.active = True - self.t = threading.Thread(target = self._run) - - # kill this thread on exit and don't add it to the join list - self.t.setDaemon(True) - self.t.start() - - self.connected = True - - rc = self.barrier() - if not rc: - self.disconnect() - return rc - - return RC_OK() - - - - - # disconnect - def disconnect (self): - if not self.connected: - return - - # signal that the context was destroyed (exit the thread loop) - self.context.term() - - # mark for join and join - self.active = False - self.t.join() - - # done - self.connected = False - - - # thread function - def _run (self): - - # socket must be created on the same thread - self.socket.setsockopt(zmq.SUBSCRIBE, '') - self.socket.setsockopt(zmq.RCVTIMEO, 5000) - self.socket.connect(self.tr) - - got_data = False - - while self.active: - try: - - line = self.socket.recv_string() - self.last_data_recv_ts = time.time() - - # signal once - if not got_data: - self.event_handler.on_async_alive() - got_data = True - - - # got a timeout - mark as not alive and retry - except zmq.Again: - - # signal once - if got_data: - self.event_handler.on_async_dead() - got_data = False - - continue - - except zmq.ContextTerminated: - # outside thread signaled us to exit - break - - msg = json.loads(line) - - name = msg['name'] - data = msg['data'] - type = msg['type'] - self.raw_snapshot[name] = data - - self.__dispatch(name, type, data) - - - # closing of socket must be from the same thread - self.socket.close(linger = 0) - - - # did we get info for the last 3 seconds ? - def is_alive (self): - if self.last_data_recv_ts == None: - return False - - return ( (time.time() - self.last_data_recv_ts) < 3 ) - - def get_stats (self): - return self.stats - - def get_raw_snapshot (self): - return self.raw_snapshot - - # dispatch the message to the right place - def __dispatch (self, name, type, data): - # stats - if name == "trex-global": - self.event_handler.handle_async_stats_update(data) - - # events - elif name == "trex-event": - self.event_handler.handle_async_event(type, data) - - # barriers - elif name == "trex-barrier": - self.handle_async_barrier(type, data) - else: - pass - - - # async barrier handling routine - def handle_async_barrier (self, type, data): - if self.async_barrier['key'] == type: - self.async_barrier['ack'] = True - - - # block on barrier for async channel - def barrier(self, timeout = 5): - - # set a random key - key = random.getrandbits(32) - self.async_barrier = {'key': key, 'ack': False} - - # expr time - expr = time.time() + timeout - - while not self.async_barrier['ack']: - - # inject - rc = self.stateless_client._transmit("publish_now", params = {'key' : key}) - if not rc: - return rc - - # fast loop - for i in xrange(0, 100): - if self.async_barrier['ack']: - break - time.sleep(0.001) - - if time.time() > expr: - return RC_ERR("*** [subscriber] - timeout - no data flow from server at : " + self.tr) - - return RC_OK() - - - diff --git a/scripts/automation/trex_control_plane/client/trex_port.py b/scripts/automation/trex_control_plane/client/trex_port.py deleted file mode 100644 index eaf64ac2..00000000 --- a/scripts/automation/trex_control_plane/client/trex_port.py +++ /dev/null @@ -1,512 +0,0 @@ - -from collections import namedtuple, OrderedDict -from common.trex_types import * -from common import trex_stats -from client_utils import packet_builder - -StreamOnPort = namedtuple('StreamOnPort', ['compiled_stream', 'metadata']) - -########## utlity ############ -def mult_to_factor (mult, max_bps_l2, max_pps, line_util): - if mult['type'] == 'raw': - return mult['value'] - - if mult['type'] == 'bps': - return mult['value'] / max_bps_l2 - - if mult['type'] == 'pps': - return mult['value'] / max_pps - - if mult['type'] == 'percentage': - return mult['value'] / line_util - - -# describes a single port -class Port(object): - STATE_DOWN = 0 - STATE_IDLE = 1 - STATE_STREAMS = 2 - STATE_TX = 3 - STATE_PAUSE = 4 - PortState = namedtuple('PortState', ['state_id', 'state_name']) - STATES_MAP = {STATE_DOWN: "DOWN", - STATE_IDLE: "IDLE", - STATE_STREAMS: "IDLE", - STATE_TX: "ACTIVE", - STATE_PAUSE: "PAUSE"} - - - def __init__ (self, port_id, speed, driver, user, comm_link, session_id): - self.port_id = port_id - self.state = self.STATE_IDLE - self.handler = None - self.comm_link = comm_link - self.transmit = comm_link.transmit - self.transmit_batch = comm_link.transmit_batch - self.user = user - self.driver = driver - self.speed = speed - self.streams = {} - self.profile = None - self.session_id = session_id - - self.port_stats = trex_stats.CPortStats(self) - - - def err(self, msg): - return RC_ERR("port {0} : {1}".format(self.port_id, msg)) - - def ok(self, data = ""): - return RC_OK(data) - - def get_speed_bps (self): - return (self.speed * 1000 * 1000 * 1000) - - # take the port - def acquire(self, force = False): - params = {"port_id": self.port_id, - "user": self.user, - "session_id": self.session_id, - "force": force} - - command = RpcCmdData("acquire", params) - rc = self.transmit(command.method, command.params) - if rc.good(): - self.handler = rc.data() - return self.ok() - else: - return self.err(rc.err()) - - # release the port - def release(self): - params = {"port_id": self.port_id, - "handler": self.handler} - - command = RpcCmdData("release", params) - rc = self.transmit(command.method, command.params) - self.handler = None - - if rc.good(): - return self.ok() - else: - return self.err(rc.err()) - - def is_acquired(self): - return (self.handler != None) - - def is_active(self): - return(self.state == self.STATE_TX ) or (self.state == self.STATE_PAUSE) - - def is_transmitting (self): - return (self.state == self.STATE_TX) - - def is_paused (self): - return (self.state == self.STATE_PAUSE) - - - def sync(self): - params = {"port_id": self.port_id} - - command = RpcCmdData("get_port_status", params) - rc = self.transmit(command.method, command.params) - if rc.bad(): - return self.err(rc.err()) - - # sync the port - port_state = rc.data()['state'] - - if port_state == "DOWN": - self.state = self.STATE_DOWN - elif port_state == "IDLE": - self.state = self.STATE_IDLE - elif port_state == "STREAMS": - self.state = self.STATE_STREAMS - elif port_state == "TX": - self.state = self.STATE_TX - elif port_state == "PAUSE": - self.state = self.STATE_PAUSE - else: - raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state)) - - # TODO: handle syncing the streams into stream_db - - return self.ok() - - - # return TRUE if write commands - def is_port_writable (self): - # operations on port can be done on state idle or state streams - return ((self.state == self.STATE_IDLE) or (self.state == self.STATE_STREAMS)) - - - # add streams - def add_streams (self, streams_list): - - if not self.is_acquired(): - return self.err("port is not owned") - - if not self.is_port_writable(): - return self.err("Please stop port before attempting to add streams") - - batch = [] - for stream in (streams_list if isinstance(streams_list, list) else [streams_list]): - - params = {"handler": self.handler, - "port_id": self.port_id, - "stream_id": stream.get_id(), - "stream": stream.to_json()} - - cmd = RpcCmdData('add_stream', params) - batch.append(cmd) - - # meta data for show streams - self.streams[stream.get_id()] = StreamOnPort(stream.to_json(), - Port._generate_stream_metadata(stream.get_id(), stream.to_json())) - - rc = self.transmit_batch(batch) - if not rc: - return self.err(rc.err()) - - - - # the only valid state now - self.state = self.STATE_STREAMS - - return self.ok() - - - - # remove stream from port - def remove_streams (self, stream_id_list): - - if not self.is_acquired(): - return self.err("port is not owned") - - if not self.is_port_writable(): - return self.err("Please stop port before attempting to remove streams") - - # single element to list - stream_id_list = stream_id_list if isinstance(stream_id_list, list) else [stream_id_list] - - # verify existance - if not all([stream_id in self.streams for stream_id in stream_id_list]): - return self.err("stream {0} does not exists".format(stream_id)) - - batch = [] - - for stream_id in stream_id_list: - params = {"handler": self.handler, - "port_id": self.port_id, - "stream_id": stream_id} - - cmd = RpcCmdData('remove_stream', params) - batch.append(cmd) - - del self.streams[stream_id] - - - rc = self.transmit_batch(batch) - if not rc: - return self.err(rc.err()) - - self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE - - return self.ok() - - - # remove all the streams - def remove_all_streams (self): - - if not self.is_acquired(): - return self.err("port is not owned") - - if not self.is_port_writable(): - return self.err("Please stop port before attempting to remove streams") - - params = {"handler": self.handler, - "port_id": self.port_id} - - rc = self.transmit("remove_all_streams", params) - if not rc: - return self.err(rc.err()) - - self.streams = {} - - self.state = self.STATE_IDLE - - return self.ok() - - # get a specific stream - def get_stream (self, stream_id): - if stream_id in self.streams: - return self.streams[stream_id] - else: - return None - - def get_all_streams (self): - return self.streams - - # start traffic - def start (self, mul, duration, force): - if not self.is_acquired(): - return self.err("port is not owned") - - if self.state == self.STATE_DOWN: - return self.err("Unable to start traffic - port is down") - - if self.state == self.STATE_IDLE: - return self.err("Unable to start traffic - no streams attached to port") - - if self.state == self.STATE_TX: - return self.err("Unable to start traffic - port is already transmitting") - - params = {"handler": self.handler, - "port_id": self.port_id, - "mul": mul, - "duration": duration, - "force": force} - - rc = self.transmit("start_traffic", params) - if rc.bad(): - return self.err(rc.err()) - - self.state = self.STATE_TX - - return self.ok() - - # stop traffic - # with force ignores the cached state and sends the command - def stop (self, force = False): - - if not self.is_acquired(): - return self.err("port is not owned") - - # port is already stopped - if not force: - if (self.state == self.STATE_IDLE) or (self.state == self.state == self.STATE_STREAMS): - return self.ok() - - - - params = {"handler": self.handler, - "port_id": self.port_id} - - rc = self.transmit("stop_traffic", params) - if rc.bad(): - return self.err(rc.err()) - - # only valid state after stop - self.state = self.STATE_STREAMS - - return self.ok() - - def pause (self): - - if not self.is_acquired(): - return self.err("port is not owned") - - if (self.state != self.STATE_TX) : - return self.err("port is not transmitting") - - params = {"handler": self.handler, - "port_id": self.port_id} - - rc = self.transmit("pause_traffic", params) - if rc.bad(): - return self.err(rc.err()) - - # only valid state after stop - self.state = self.STATE_PAUSE - - return self.ok() - - - def resume (self): - - if not self.is_acquired(): - return self.err("port is not owned") - - if (self.state != self.STATE_PAUSE) : - return self.err("port is not in pause mode") - - params = {"handler": self.handler, - "port_id": self.port_id} - - rc = self.transmit("resume_traffic", params) - if rc.bad(): - return self.err(rc.err()) - - # only valid state after stop - self.state = self.STATE_TX - - return self.ok() - - - def update (self, mul, force): - - if not self.is_acquired(): - return self.err("port is not owned") - - if (self.state != self.STATE_TX) : - return self.err("port is not transmitting") - - params = {"handler": self.handler, - "port_id": self.port_id, - "mul": mul, - "force": force} - - rc = self.transmit("update_traffic", params) - if rc.bad(): - return self.err(rc.err()) - - return self.ok() - - - def validate (self): - - if not self.is_acquired(): - return self.err("port is not owned") - - if (self.state == self.STATE_DOWN): - return self.err("port is down") - - if (self.state == self.STATE_IDLE): - return self.err("no streams attached to port") - - params = {"handler": self.handler, - "port_id": self.port_id} - - rc = self.transmit("validate", params) - if rc.bad(): - return self.err(rc.err()) - - self.profile = rc.data() - - return self.ok() - - def get_profile (self): - return self.profile - - - def print_profile (self, mult, duration): - if not self.get_profile(): - return - - rate = self.get_profile()['rate'] - graph = self.get_profile()['graph'] - - print format_text("Profile Map Per Port\n", 'underline', 'bold') - - factor = mult_to_factor(mult, rate['max_bps_l2'], rate['max_pps'], rate['max_line_util']) - - print "Profile max BPS L2 (base / req): {:^12} / {:^12}".format(format_num(rate['max_bps_l2'], suffix = "bps"), - format_num(rate['max_bps_l2'] * factor, suffix = "bps")) - - print "Profile max BPS L1 (base / req): {:^12} / {:^12}".format(format_num(rate['max_bps_l1'], suffix = "bps"), - format_num(rate['max_bps_l1'] * factor, suffix = "bps")) - - print "Profile max PPS (base / req): {:^12} / {:^12}".format(format_num(rate['max_pps'], suffix = "pps"), - format_num(rate['max_pps'] * factor, suffix = "pps"),) - - print "Profile line util. (base / req): {:^12} / {:^12}".format(format_percentage(rate['max_line_util']), - format_percentage(rate['max_line_util'] * factor)) - - - # duration - exp_time_base_sec = graph['expected_duration'] / (1000 * 1000) - exp_time_factor_sec = exp_time_base_sec / factor - - # user configured a duration - if duration > 0: - if exp_time_factor_sec > 0: - exp_time_factor_sec = min(exp_time_factor_sec, duration) - else: - exp_time_factor_sec = duration - - - print "Duration (base / req): {:^12} / {:^12}".format(format_time(exp_time_base_sec), - format_time(exp_time_factor_sec)) - print "\n" - - - def get_port_state_name(self): - return self.STATES_MAP.get(self.state, "Unknown") - - ################# stats handler ###################### - def generate_port_stats(self): - return self.port_stats.generate_stats() - - def generate_port_status(self): - return {"type": self.driver, - "maximum": "{speed} Gb/s".format(speed=self.speed), - "status": self.get_port_state_name() - } - - def clear_stats(self): - return self.port_stats.clear_stats() - - - def get_stats (self): - return self.port_stats.get_stats() - - - def invalidate_stats(self): - return self.port_stats.invalidate() - - ################# stream printout ###################### - def generate_loaded_streams_sum(self, stream_id_list): - if self.state == self.STATE_DOWN or self.state == self.STATE_STREAMS: - return {} - streams_data = {} - - if not stream_id_list: - # if no mask has been provided, apply to all streams on port - stream_id_list = self.streams.keys() - - - streams_data = {stream_id: self.streams[stream_id].metadata.get('stream_sum', ["N/A"] * 6) - for stream_id in stream_id_list - if stream_id in self.streams} - - - return {"referring_file" : "", - "streams" : streams_data} - - @staticmethod - def _generate_stream_metadata(stream_id, compiled_stream_obj): - meta_dict = {} - # create packet stream description - pkt_bld_obj = packet_builder.CTRexPktBuilder() - pkt_bld_obj.load_from_stream_obj(compiled_stream_obj) - # generate stream summary based on that - - next_stream = "None" if compiled_stream_obj['next_stream_id']==-1 else compiled_stream_obj['next_stream_id'] - - meta_dict['stream_sum'] = OrderedDict([("id", stream_id), - ("packet_type", "/".join(pkt_bld_obj.get_packet_layers())), - ("length", pkt_bld_obj.get_packet_length()), - ("mode", compiled_stream_obj['mode']['type']), - ("rate_pps", compiled_stream_obj['mode']['pps']), - ("next_stream", next_stream) - ]) - return meta_dict - - ################# events handler ###################### - def async_event_port_stopped (self): - self.state = self.STATE_STREAMS - - - def async_event_port_started (self): - self.state = self.STATE_TX - - - def async_event_port_paused (self): - self.state = self.STATE_PAUSE - - - def async_event_port_resumed (self): - self.state = self.STATE_TX - - def async_event_forced_acquired (self): - self.handler = None - diff --git a/scripts/automation/trex_control_plane/client/trex_stateless_client.py b/scripts/automation/trex_control_plane/client/trex_stateless_client.py deleted file mode 100755 index 95fd2a69..00000000 --- a/scripts/automation/trex_control_plane/client/trex_stateless_client.py +++ /dev/null @@ -1,2035 +0,0 @@ -#!/router/bin/python - -try: - # support import for Python 2 - import outer_packages -except ImportError: - # support import for Python 3 - import client.outer_packages - -from client_utils.jsonrpc_client import JsonRpcClient, BatchMessage -from client_utils import general_utils -from client_utils.packet_builder import CTRexPktBuilder -import json - -from common.trex_streams import * -from collections import namedtuple -from common.text_opts import * -from common import trex_stats -from client_utils import parsing_opts, text_tables -import time -import datetime -import re -import random -from trex_port import Port -from common.trex_types import * -from common.trex_stl_exceptions import * -from trex_async_client import CTRexAsyncClient -from yaml import YAMLError - - - -############################ logger ############################# -############################ ############################# -############################ ############################# - -# logger API for the client -class LoggerApi(object): - # verbose levels - VERBOSE_QUIET = 0 - VERBOSE_REGULAR = 1 - VERBOSE_HIGH = 2 - - def __init__(self): - self.level = LoggerApi.VERBOSE_REGULAR - - # implemented by specific logger - def write(self, msg, newline = True): - raise Exception("implement this") - - # implemented by specific logger - def flush(self): - raise Exception("implement this") - - def set_verbose (self, level): - if not level in xrange(self.VERBOSE_QUIET, self.VERBOSE_HIGH + 1): - raise ValueError("bad value provided for logger") - - self.level = level - - def get_verbose (self): - return self.level - - - def check_verbose (self, level): - return (self.level >= level) - - - # simple log message with verbose - def log (self, msg, level = VERBOSE_REGULAR, newline = True): - if not self.check_verbose(level): - return - - self.write(msg, newline) - - # logging that comes from async event - def async_log (self, msg, level = VERBOSE_REGULAR, newline = True): - self.log(msg, level, newline) - - - def pre_cmd (self, desc): - self.log(format_text('\n{:<60}'.format(desc), 'bold'), newline = False) - self.flush() - - def post_cmd (self, rc): - if rc: - self.log(format_text("[SUCCESS]\n", 'green', 'bold')) - else: - self.log(format_text("[FAILED]\n", 'red', 'bold')) - - - def log_cmd (self, desc): - self.pre_cmd(desc) - self.post_cmd(True) - - - # supress object getter - def supress (self): - class Supress(object): - def __init__ (self, logger): - self.logger = logger - - def __enter__ (self): - self.saved_level = self.logger.get_verbose() - self.logger.set_verbose(LoggerApi.VERBOSE_QUIET) - - def __exit__ (self, type, value, traceback): - self.logger.set_verbose(self.saved_level) - - return Supress(self) - - - -# default logger - to stdout -class DefaultLogger(LoggerApi): - - def __init__ (self): - super(DefaultLogger, self).__init__() - - def write (self, msg, newline = True): - if newline: - print msg - else: - print msg, - - def flush (self): - sys.stdout.flush() - - -############################ async event hander ############################# -############################ ############################# -############################ ############################# - -# handles different async events given to the client -class AsyncEventHandler(object): - - def __init__ (self, client): - self.client = client - self.logger = self.client.logger - - self.events = [] - - # public functions - - def get_events (self): - return self.events - - - def clear_events (self): - self.events = [] - - - def on_async_dead (self): - if self.client.connected: - msg = 'lost connection to server' - self.__add_event_log(msg, 'local', True) - self.client.connected = False - - - def on_async_alive (self): - pass - - - # handles an async stats update from the subscriber - def handle_async_stats_update(self, dump_data): - global_stats = {} - port_stats = {} - - # filter the values per port and general - for key, value in dump_data.iteritems(): - # match a pattern of ports - m = re.search('(.*)\-([0-8])', key) - if m: - port_id = int(m.group(2)) - field_name = m.group(1) - if self.client.ports.has_key(port_id): - if not port_id in port_stats: - port_stats[port_id] = {} - port_stats[port_id][field_name] = value - else: - continue - else: - # no port match - general stats - global_stats[key] = value - - # update the general object with the snapshot - self.client.global_stats.update(global_stats) - - # update all ports - for port_id, data in port_stats.iteritems(): - self.client.ports[port_id].port_stats.update(data) - - - # dispatcher for server async events (port started, port stopped and etc.) - def handle_async_event (self, type, data): - # DP stopped - - show_event = False - - # port started - if (type == 0): - port_id = int(data['port_id']) - ev = "Port {0} has started".format(port_id) - self.__async_event_port_started(port_id) - - # port stopped - elif (type == 1): - port_id = int(data['port_id']) - ev = "Port {0} has stopped".format(port_id) - - # call the handler - self.__async_event_port_stopped(port_id) - - - # port paused - elif (type == 2): - port_id = int(data['port_id']) - ev = "Port {0} has paused".format(port_id) - - # call the handler - self.__async_event_port_paused(port_id) - - # port resumed - elif (type == 3): - port_id = int(data['port_id']) - ev = "Port {0} has resumed".format(port_id) - - # call the handler - self.__async_event_port_resumed(port_id) - - # port finished traffic - elif (type == 4): - port_id = int(data['port_id']) - ev = "Port {0} job done".format(port_id) - - # call the handler - self.__async_event_port_stopped(port_id) - show_event = True - - # port was stolen... - elif (type == 5): - session_id = data['session_id'] - - # false alarm, its us - if session_id == self.client.session_id: - return - - port_id = int(data['port_id']) - who = data['who'] - - ev = "Port {0} was forcely taken by '{1}'".format(port_id, who) - - # call the handler - self.__async_event_port_forced_acquired(port_id) - show_event = True - - # server stopped - elif (type == 100): - ev = "Server has stopped" - self.__async_event_server_stopped() - show_event = True - - - else: - # unknown event - ignore - return - - - self.__add_event_log(ev, 'server', show_event) - - - # private functions - - def __async_event_port_stopped (self, port_id): - self.client.ports[port_id].async_event_port_stopped() - - - def __async_event_port_started (self, port_id): - self.client.ports[port_id].async_event_port_started() - - - def __async_event_port_paused (self, port_id): - self.client.ports[port_id].async_event_port_paused() - - - def __async_event_port_resumed (self, port_id): - self.client.ports[port_id].async_event_port_resumed() - - - def __async_event_port_forced_acquired (self, port_id): - self.client.ports[port_id].async_event_forced_acquired() - - - def __async_event_server_stopped (self): - self.client.connected = False - - - # add event to log - def __add_event_log (self, msg, ev_type, show = False): - - if ev_type == "server": - prefix = "[server]" - elif ev_type == "local": - prefix = "[local]" - - ts = time.time() - st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold'))) - - if show: - self.logger.async_log(format_text("\n\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))) - - - - - -############################ RPC layer ############################# -############################ ############################# -############################ ############################# - -class CCommLink(object): - """describes the connectivity of the stateless client method""" - def __init__(self, server="localhost", port=5050, virtual=False, prn_func = None): - self.virtual = virtual - self.server = server - self.port = port - self.rpc_link = JsonRpcClient(self.server, self.port, prn_func) - - @property - def is_connected(self): - if not self.virtual: - return self.rpc_link.connected - else: - return True - - def get_server (self): - return self.server - - def get_port (self): - return self.port - - def connect(self): - if not self.virtual: - return self.rpc_link.connect() - - def disconnect(self): - if not self.virtual: - return self.rpc_link.disconnect() - - def transmit(self, method_name, params={}): - if self.virtual: - self._prompt_virtual_tx_msg() - _, msg = self.rpc_link.create_jsonrpc_v2(method_name, params) - print msg - return - else: - return self.rpc_link.invoke_rpc_method(method_name, params) - - def transmit_batch(self, batch_list): - if self.virtual: - self._prompt_virtual_tx_msg() - print [msg - for _, msg in [self.rpc_link.create_jsonrpc_v2(command.method, command.params) - for command in batch_list]] - else: - batch = self.rpc_link.create_batch() - for command in batch_list: - batch.add(command.method, command.params) - # invoke the batch - return batch.invoke() - - def _prompt_virtual_tx_msg(self): - print "Transmitting virtually over tcp://{server}:{port}".format(server=self.server, - port=self.port) - - - -############################ client ############################# -############################ ############################# -############################ ############################# - -class STLClient(object): - """docstring for STLClient""" - - def __init__(self, - username = general_utils.get_current_user(), - server = "localhost", - sync_port = 4501, - async_port = 4500, - verbose_level = LoggerApi.VERBOSE_QUIET, - logger = None, - virtual = False): - - - self.username = username - - # init objects - self.ports = {} - self.server_version = {} - self.system_info = {} - self.session_id = random.getrandbits(32) - self.connected = False - - # logger - self.logger = DefaultLogger() if not logger else logger - - # initial verbose - self.logger.set_verbose(verbose_level) - - # low level RPC layer - self.comm_link = CCommLink(server, - sync_port, - virtual, - self.logger) - - # async event handler manager - self.event_handler = AsyncEventHandler(self) - - # async subscriber level - self.async_client = CTRexAsyncClient(server, - async_port, - self) - - - - - # stats - self.connection_info = {"username": username, - "server": server, - "sync_port": sync_port, - "async_port": async_port, - "virtual": virtual} - - - self.global_stats = trex_stats.CGlobalStats(self.connection_info, - self.server_version, - self.ports) - - self.stats_generator = trex_stats.CTRexInfoGenerator(self.global_stats, - self.ports) - - - - ############# private functions - used by the class itself ########### - - # some preprocessing for port argument - def __ports (self, port_id_list): - - # none means all - if port_id_list == None: - return range(0, self.get_port_count()) - - # always list - if isinstance(port_id_list, int): - port_id_list = [port_id_list] - - if not isinstance(port_id_list, list): - raise ValueError("bad port id list: {0}".format(port_id_list)) - - for port_id in port_id_list: - if not isinstance(port_id, int) or (port_id < 0) or (port_id > self.get_port_count()): - raise ValueError("bad port id {0}".format(port_id)) - - return port_id_list - - - # sync ports - def __sync_ports (self, port_id_list = None, force = False): - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].sync()) - - return rc - - # acquire ports, if port_list is none - get all - def __acquire (self, port_id_list = None, force = False): - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].acquire(force)) - - return rc - - # release ports - def __release (self, port_id_list = None): - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].release()) - - return rc - - - def __add_streams(self, stream_list, port_id_list = None): - - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].add_streams(stream_list)) - - return rc - - - - def __remove_streams(self, stream_id_list, port_id_list = None): - - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].remove_streams(stream_id_list)) - - return rc - - - - def __remove_all_streams(self, port_id_list = None): - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].remove_all_streams()) - - return rc - - - def __get_stream(self, stream_id, port_id, get_pkt = False): - - return self.ports[port_id].get_stream(stream_id) - - - def __get_all_streams(self, port_id, get_pkt = False): - - return self.ports[port_id].get_all_streams() - - - def __get_stream_id_list(self, port_id): - - return self.ports[port_id].get_stream_id_list() - - - def __start (self, multiplier, duration, port_id_list = None, force = False): - - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].start(multiplier, duration, force)) - - return rc - - - def __resume (self, port_id_list = None, force = False): - - port_id_list = self.__ports(port_id_list) - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].resume()) - - return rc - - def __pause (self, port_id_list = None, force = False): - - port_id_list = self.__ports(port_id_list) - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].pause()) - - return rc - - - def __stop (self, port_id_list = None, force = False): - - port_id_list = self.__ports(port_id_list) - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].stop(force)) - - return rc - - - def __update (self, mult, port_id_list = None, force = False): - - port_id_list = self.__ports(port_id_list) - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].update(mult, force)) - - return rc - - - def __validate (self, port_id_list = None): - port_id_list = self.__ports(port_id_list) - - rc = RC() - - for port_id in port_id_list: - rc.add(self.ports[port_id].validate()) - - return rc - - - - # connect to server - def __connect(self): - - # first disconnect if already connected - if self.is_connected(): - self.__disconnect() - - # clear this flag - self.connected = False - - # connect sync channel - self.logger.pre_cmd("Connecting to RPC server on {0}:{1}".format(self.connection_info['server'], self.connection_info['sync_port'])) - rc = self.comm_link.connect() - self.logger.post_cmd(rc) - - if not rc: - return rc - - # version - rc = self._transmit("get_version") - if not rc: - return rc - - - self.server_version = rc.data() - self.global_stats.server_version = rc.data() - - # cache system info - rc = self._transmit("get_system_info") - if not rc: - return rc - - self.system_info = rc.data() - - # cache supported commands - rc = self._transmit("get_supported_cmds") - if not rc: - return rc - - self.supported_cmds = rc.data() - - # create ports - for port_id in xrange(self.system_info["port_count"]): - speed = self.system_info['ports'][port_id]['speed'] - driver = self.system_info['ports'][port_id]['driver'] - - self.ports[port_id] = Port(port_id, - speed, - driver, - self.username, - self.comm_link, - self.session_id) - - - # sync the ports - rc = self.__sync_ports() - if not rc: - return rc - - - # connect async channel - self.logger.pre_cmd("connecting to publisher server on {0}:{1}".format(self.connection_info['server'], self.connection_info['async_port'])) - rc = self.async_client.connect() - self.logger.post_cmd(rc) - - if not rc: - return rc - - self.connected = True - - return RC_OK() - - - # disconenct from server - def __disconnect(self, release_ports = True): - # release any previous acquired ports - if self.is_connected() and release_ports: - self.__release(self.get_acquired_ports()) - - self.comm_link.disconnect() - self.async_client.disconnect() - - self.connected = False - - return RC_OK() - - - # clear stats - def __clear_stats(self, port_id_list, clear_global): - - for port_id in port_id_list: - self.ports[port_id].clear_stats() - - if clear_global: - self.global_stats.clear_stats() - - self.logger.log_cmd("clearing stats on port(s) {0}:".format(port_id_list)) - - return RC - - - # get stats - def __get_stats (self, port_id_list): - stats = {} - - stats['global'] = self.global_stats.get_stats() - - total = {} - for port_id in port_id_list: - port_stats = self.ports[port_id].get_stats() - stats[port_id] = port_stats - - for k, v in port_stats.iteritems(): - if not k in total: - total[k] = v - else: - total[k] += v - - stats['total'] = total - - return stats - - - ############ functions used by other classes but not users ############## - - def _verify_port_id_list (self, port_id_list): - # check arguments - if not isinstance(port_id_list, list): - return RC_ERR("ports should be an instance of 'list' not {0}".format(type(port_id_list))) - - # all ports are valid ports - if not port_id_list or not all([port_id in self.get_all_ports() for port_id in port_id_list]): - return RC_ERR("") - - return RC_OK() - - def _validate_port_list(self, port_id_list): - if not isinstance(port_id_list, list): - return False - - # check each item of the sequence - return (port_id_list and all([port_id in self.get_all_ports() for port_id in port_id_list])) - - - - # transmit request on the RPC link - def _transmit(self, method_name, params={}): - return self.comm_link.transmit(method_name, params) - - # transmit batch request on the RPC link - def _transmit_batch(self, batch_list): - return self.comm_link.transmit_batch(batch_list) - - # stats - def _get_formatted_stats(self, port_id_list, stats_mask=set()): - stats_opts = trex_stats.ALL_STATS_OPTS.intersection(stats_mask) - - stats_obj = {} - for stats_type in stats_opts: - stats_obj.update(self.stats_generator.generate_single_statistic(port_id_list, stats_type)) - - return stats_obj - - def _get_streams(self, port_id_list, streams_mask=set()): - - streams_obj = self.stats_generator.generate_streams_info(port_id_list, streams_mask) - - return streams_obj - - - def _invalidate_stats (self, port_id_list): - for port_id in port_id_list: - self.ports[port_id].invalidate_stats() - - self.global_stats.invalidate() - - return RC_OK() - - - - - - ################################# - # ------ private methods ------ # - @staticmethod - def __get_mask_keys(ok_values={True}, **kwargs): - masked_keys = set() - for key, val in kwargs.iteritems(): - if val in ok_values: - masked_keys.add(key) - return masked_keys - - @staticmethod - def __filter_namespace_args(namespace, ok_values): - return {k: v for k, v in namespace.__dict__.items() if k in ok_values} - - - # API decorator - double wrap because of argument - def __api_check(connected = True): - - def wrap (f): - def wrap2(*args, **kwargs): - client = args[0] - - func_name = f.__name__ - - # check connection - if connected and not client.is_connected(): - raise STLStateError(func_name, 'disconnected') - - ret = f(*args, **kwargs) - return ret - return wrap2 - - return wrap - - - - ############################ API ############################# - ############################ ############################# - ############################ ############################# - def __enter__ (self): - self.connect() - self.acquire(force = True) - self.reset() - return self - - def __exit__ (self, type, value, traceback): - if self.get_active_ports(): - self.stop(self.get_active_ports()) - self.disconnect() - - ############################ Getters ############################# - ############################ ############################# - ############################ ############################# - - - # return verbose level of the logger - def get_verbose (self): - return self.logger.get_verbose() - - # is the client on read only mode ? - def is_all_ports_acquired (self): - return not (self.get_all_ports() == self.get_acquired_ports()) - - # is the client connected ? - def is_connected (self): - return self.connected and self.comm_link.is_connected - - - # get connection info - def get_connection_info (self): - return self.connection_info - - - # get supported commands by the server - def get_server_supported_cmds(self): - return self.supported_cmds - - # get server version - def get_server_version(self): - return self.server_version - - # get server system info - def get_server_system_info(self): - return self.system_info - - # get port count - def get_port_count(self): - return len(self.ports) - - - # returns the port object - def get_port (self, port_id): - port = self.ports.get(port_id, None) - if (port != None): - return port - else: - raise STLArgumentError('port id', port_id, valid_values = self.get_all_ports()) - - - # get all ports as IDs - def get_all_ports (self): - return self.ports.keys() - - # get all acquired ports - def get_acquired_ports(self): - return [port_id - for port_id, port_obj in self.ports.iteritems() - if port_obj.is_acquired()] - - # get all active ports (TX or pause) - def get_active_ports(self): - return [port_id - for port_id, port_obj in self.ports.iteritems() - if port_obj.is_active()] - - # get paused ports - def get_paused_ports (self): - return [port_id - for port_id, port_obj in self.ports.iteritems() - if port_obj.is_paused()] - - # get all TX ports - def get_transmitting_ports (self): - return [port_id - for port_id, port_obj in self.ports.iteritems() - if port_obj.is_transmitting()] - - - # get stats - def get_stats (self, ports = None, async_barrier = True): - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - else: - ports = self.__ports(ports) - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # check async barrier - if not type(async_barrier) is bool: - raise STLArgumentError('async_barrier', async_barrier) - - - # if the user requested a barrier - use it - if async_barrier: - rc = self.async_client.barrier() - if not rc: - raise STLError(rc) - - return self.__get_stats(ports) - - # return all async events - def get_events (self): - return self.event_handler.get_events() - - ############################ Commands ############################# - ############################ ############################# - ############################ ############################# - - - """ - Sets verbose level - - :parameters: - level : str - "high" - "low" - "normal" - - :raises: - None - - """ - def set_verbose (self, level): - modes = {'low' : LoggerApi.VERBOSE_QUIET, 'normal': LoggerApi.VERBOSE_REGULAR, 'high': LoggerApi.VERBOSE_HIGH} - - if not level in modes.keys(): - raise STLArgumentError('level', level) - - self.logger.set_verbose(modes[level]) - - - """ - Connects to the TRex server - - :parameters: - None - - :raises: - + :exc:`STLError` - - """ - @__api_check(False) - def connect (self): - rc = self.__connect() - if not rc: - raise STLError(rc) - - - """ - Disconnects from the server - - :parameters: - stop_traffic : bool - tries to stop traffic before disconnecting - release_ports : bool - tries to release all the acquired ports - - """ - @__api_check(False) - def disconnect (self, stop_traffic = True, release_ports = True): - - # try to stop ports but do nothing if not possible - if stop_traffic: - try: - self.stop() - except STLError: - pass - - - self.logger.pre_cmd("Disconnecting from server at '{0}':'{1}'".format(self.connection_info['server'], - self.connection_info['sync_port'])) - rc = self.__disconnect(release_ports) - self.logger.post_cmd(rc) - - - - """ - Acquires ports for executing commands - - :parameters: - ports : list - ports to execute the command - force : bool - force acquire the ports - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def acquire (self, ports = None, force = False): - # by default use all ports - if ports == None: - ports = self.get_all_ports() - - # verify ports - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # verify valid port id list - if force: - self.logger.pre_cmd("Force acquiring ports {0}:".format(ports)) - else: - self.logger.pre_cmd("Acquiring ports {0}:".format(ports)) - - rc = self.__acquire(ports, force) - - self.logger.post_cmd(rc) - - if not rc: - # cleanup - self.__release(ports) - raise STLError(rc) - - - """ - Release ports - - :parameters: - ports : list - ports to execute the command - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def release (self, ports = None): - # by default use all acquired ports - if ports == None: - ports = self.get_acquired_ports() - - # verify ports - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.logger.pre_cmd("Releasing ports {0}:".format(ports)) - rc = self.__release(ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - """ - Pings the server - - :parameters: - None - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def ping(self): - self.logger.pre_cmd( "Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'], - self.connection_info['sync_port'])) - rc = self._transmit("ping") - - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - - """ - force acquire ports, stop the traffic, remove all streams and clear stats - - :parameters: - ports : list - ports to execute the command - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def reset(self, ports = None): - - # by default use all ports - if ports == None: - ports = self.get_all_ports() - - # verify ports - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.acquire(ports, force = True) - self.stop(ports) - self.remove_all_streams(ports) - self.clear_stats(ports) - - - """ - remove all streams from port(s) - - :parameters: - ports : list - ports to execute the command - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def remove_all_streams (self, ports = None): - - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.logger.pre_cmd("Removing all streams from port(s) {0}:".format(ports)) - rc = self.__remove_all_streams(ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - """ - add a list of streams to port(s) - - :parameters: - ports : list - ports to execute the command - streams: list - streams to attach - - :returns: - list of stream IDs in order of the stream list - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def add_streams (self, streams, ports = None): - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # transform single stream - if not isinstance(streams, list): - streams = [streams] - - # check streams - if not all([isinstance(stream, STLStream) for stream in streams]): - raise STLArgumentError('streams', streams) - - self.logger.pre_cmd("Attaching {0} streams to port(s) {1}:".format(len(streams), ports)) - rc = self.__add_streams(streams, ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - return [stream.get_id() for stream in streams] - - - """ - remove a list of streams from ports - - :parameters: - ports : list - ports to execute the command - stream_id_list: list - stream id list to remove - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def remove_streams (self, stream_id_list, ports = None): - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # transform single stream - if not isinstance(stream_id_list, list): - stream_id_list = [stream_id_list] - - # check streams - if not all([isinstance(stream_id, long) for stream_id in stream_id_list]): - raise STLArgumentError('stream_id_list', stream_id_list) - - # remove streams - self.logger.pre_cmd("Removing {0} streams from port(s) {1}:".format(len(stream_id_list), ports)) - rc = self.__remove_streams(stream_id_list, ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - """ - load a profile file to port(s) - - :parameters: - filename : str - filename to load - ports : list - ports to execute the command - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def load_profile (self, filename, ports = None): - - # check filename - if not os.path.isfile(filename): - raise STLError("file '{0}' does not exists".format(filename)) - - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - - streams = None - - # try YAML - try: - streams_db = CStreamsDB() - stream_list = streams_db.load_yaml_file(filename) - # convert to new style stream object - streams = [HACKSTLStream(stream) for stream in stream_list.compiled] - except YAMLError: - # try python loader - try: - basedir = os.path.dirname(filename) - - sys.path.append(basedir) - file = os.path.basename(filename).split('.')[0] - module = __import__(file, globals(), locals(), [], -1) - reload(module) # reload the update - - streams = module.register().get_streams() - - except Exception as e : - print str(e); - traceback.print_exc(file=sys.stdout) - raise STLError("Unexpected error: '{0}'".format(filename)) - - - self.add_streams(streams, ports) - - - - """ - start traffic on port(s) - - :parameters: - ports : list - ports to execute command - - mult : str - multiplier in a form of pps, bps, or line util in % - examples: "5kpps", "10gbps", "85%", "32mbps" - - force : bool - imply stopping the port of active and also - forces a profile that exceeds the L1 BW - - duration : int - limit the run for time in seconds - -1 means unlimited - - total : bool - should the B/W be divided by the ports - or duplicated for each - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def start (self, - ports = None, - mult = "1", - force = False, - duration = -1, - total = False): - - - # by default use all ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # verify multiplier - mult_obj = parsing_opts.decode_multiplier(mult, - allow_update = False, - divide_count = len(ports) if total else 1) - if not mult_obj: - raise STLArgumentError('mult', mult) - - # some type checkings - - if not type(force) is bool: - raise STLArgumentError('force', force) - - if not isinstance(duration, (int, float)): - raise STLArgumentError('duration', duration) - - if not type(total) is bool: - raise STLArgumentError('total', total) - - - # verify ports are stopped or force stop them - active_ports = list(set(self.get_active_ports()).intersection(ports)) - if active_ports: - if not force: - raise STLError("Port(s) {0} are active - please stop them or specify 'force'".format(active_ports)) - else: - rc = self.stop(active_ports) - if not rc: - raise STLError(rc) - - - # start traffic - self.logger.pre_cmd("Starting traffic on port(s) {0}:".format(ports)) - rc = self.__start(mult_obj, duration, ports, force) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - - - """ - stop port(s) - - :parameters: - ports : list - ports to execute the command - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def stop (self, ports = None): - - # by default the user means all the active ports - if ports == None: - ports = self.get_active_ports() - if not ports: - return - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.logger.pre_cmd("Stopping traffic on port(s) {0}:".format(ports)) - rc = self.__stop(ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - - """ - update traffic on port(s) - - :parameters: - ports : list - ports to execute command - - mult : str - multiplier in a form of pps, bps, or line util in % - and also with +/- - examples: "5kpps+", "10gbps-", "85%", "32mbps", "20%+" - - force : bool - forces a profile that exceeds the L1 BW - - total : bool - should the B/W be divided by the ports - or duplicated for each - - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def update (self, ports = None, mult = "1", total = False, force = False): - - # by default the user means all the active ports - if ports == None: - ports = self.get_active_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # verify multiplier - mult_obj = parsing_opts.decode_multiplier(mult, - allow_update = True, - divide_count = len(ports) if total else 1) - if not mult_obj: - raise STLArgumentError('mult', mult) - - # verify total - if not type(total) is bool: - raise STLArgumentError('total', total) - - - # call low level functions - self.logger.pre_cmd("Updating traffic on port(s) {0}:".format(ports)) - rc = self.__update(mult, ports, force) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - - """ - pause traffic on port(s) - - :parameters: - ports : list - ports to execute command - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def pause (self, ports = None): - - # by default the user means all the TX ports - if ports == None: - ports = self.get_transmitting_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.logger.pre_cmd("Pausing traffic on port(s) {0}:".format(ports)) - rc = self.__pause(ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - - """ - resume traffic on port(s) - - :parameters: - ports : list - ports to execute command - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def resume (self, ports = None): - - # by default the user means all the paused ports - if ports == None: - ports = self.get_paused_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - self.logger.pre_cmd("Resume traffic on port(s) {0}:".format(ports)) - rc = self.__resume(ports) - self.logger.post_cmd(rc) - - if not rc: - raise STLError(rc) - - - """ - validate port(s) configuration - - :parameters: - ports : list - ports to execute command - - mult : str - multiplier in a form of pps, bps, or line util in % - examples: "5kpps", "10gbps", "85%", "32mbps" - - duration : int - limit the run for time in seconds - -1 means unlimited - - total : bool - should the B/W be divided by the ports - or duplicated for each - - :raises: - + :exc:`STLError` - - """ - @__api_check(True) - def validate (self, ports = None, mult = "1", duration = "-1", total = False): - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # verify multiplier - mult_obj = parsing_opts.decode_multiplier(mult, - allow_update = True, - divide_count = len(ports) if total else 1) - if not mult_obj: - raise STLArgumentError('mult', mult) - - - if not isinstance(duration, (int, float)): - raise STLArgumentError('duration', duration) - - - self.logger.pre_cmd("Validating streams on port(s) {0}:".format(ports)) - rc = self.__validate(ports) - self.logger.post_cmd(rc) - - - for port in ports: - self.ports[port].print_profile(mult_obj, duration) - - - """ - clear stats on port(s) - - :parameters: - ports : list - ports to execute command - - clear_global : bool - clear the global stats - - :raises: - + :exc:`STLError` - - """ - @__api_check(False) - def clear_stats (self, ports = None, clear_global = True): - - # by default use all ports - if ports == None: - ports = self.get_all_ports() - else: - ports = self.__ports(ports) - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - # verify clear global - if not type(clear_global) is bool: - raise STLArgumentError('clear_global', clear_global) - - - rc = self.__clear_stats(ports, clear_global) - if not rc: - raise STLError(rc) - - - - - - """ - block until specify port(s) traffic has ended - - :parameters: - ports : list - ports to execute command - - timeout : int - timeout in seconds - - :raises: - + :exc:`STLTimeoutError` - in case timeout has expired - + :exe:'STLError' - - """ - @__api_check(True) - def wait_on_traffic (self, ports = None, timeout = 60): - - # by default use all acquired ports - if ports == None: - ports = self.get_acquired_ports() - - # verify valid port id list - rc = self._validate_port_list(ports) - if not rc: - raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) - - expr = time.time() + timeout - - # wait while any of the required ports are active - while set(self.get_active_ports()).intersection(ports): - time.sleep(0.01) - if time.time() > expr: - raise STLTimeoutError(timeout) - - - """ - clear all events - - :parameters: - None - - :raises: - None - - """ - def clear_events (self): - self.event_handler.clear_events() - - - ############################ Line ############################# - ############################ Commands ############################# - ############################ ############################# - - # console decorator - def __console(f): - def wrap(*args): - client = args[0] - - time1 = time.time() - - try: - rc = f(*args) - except STLError as e: - client.logger.log("Log:\n" + format_text(e.brief() + "\n", 'bold')) - return - - # if got true - print time - if rc: - delta = time.time() - time1 - client.logger.log(format_time(delta) + "\n") - - - return wrap - - - @__console - def connect_line (self, line): - '''Connects to the TRex server''' - # define a parser - parser = parsing_opts.gen_parser(self, - "connect", - self.connect_line.__doc__, - parsing_opts.FORCE) - - opts = parser.parse_args(line.split()) - - if opts is None: - return - - # call the API - self.connect() - self.acquire(force = opts.force) - - # true means print time - return True - - @__console - def disconnect_line (self, line): - self.disconnect() - - - - @__console - def reset_line (self, line): - self.reset() - - # true means print time - return True - - - @__console - def start_line (self, line): - '''Start selected traffic in specified ports on TRex\n''' - # define a parser - parser = parsing_opts.gen_parser(self, - "start", - self.start_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL, - parsing_opts.TOTAL, - parsing_opts.FORCE, - parsing_opts.STREAM_FROM_PATH_OR_FILE, - parsing_opts.DURATION, - parsing_opts.MULTIPLIER_STRICT, - parsing_opts.DRY_RUN) - - opts = parser.parse_args(line.split()) - - - if opts is None: - return - - - active_ports = list(set(self.get_active_ports()).intersection(opts.ports)) - - if active_ports: - if not opts.force: - msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports) - self.logger.log(format_text(msg, 'bold')) - return - else: - self.stop(active_ports) - - - # remove all streams - self.remove_all_streams(opts.ports) - - # pack the profile - self.load_profile(opts.file[0], opts.ports) - - if opts.dry: - self.validate(opts.ports, opts.mult, opts.duration, opts.total) - else: - self.start(opts.ports, - opts.mult, - opts.force, - opts.duration, - opts.total) - - # true means print time - return True - - - - @__console - def stop_line (self, line): - '''Stop active traffic in specified ports on TRex\n''' - parser = parsing_opts.gen_parser(self, - "stop", - self.stop_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) - - if not ports: - self.logger.log(format_text("No active traffic on provided ports\n", 'bold')) - return - - self.stop(ports) - - # true means print time - return True - - - @__console - def update_line (self, line): - '''Update port(s) speed currently active\n''' - parser = parsing_opts.gen_parser(self, - "update", - self.update_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL, - parsing_opts.MULTIPLIER, - parsing_opts.TOTAL, - parsing_opts.FORCE) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - # find the relevant ports - ports = list(set(self.get_active_ports()).intersection(opts.ports)) - - if not ports: - self.logger.log(format_text("No ports in valid state to update\n", 'bold')) - return - - self.update(ports, opts.mult, opts.total, opts.force) - - # true means print time - return True - - - @__console - def pause_line (self, line): - '''Pause active traffic in specified ports on TRex\n''' - parser = parsing_opts.gen_parser(self, - "pause", - self.pause_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - # find the relevant ports - ports = list(set(self.get_transmitting_ports()).intersection(opts.ports)) - - if not ports: - self.logger.log(format_text("No ports in valid state to pause\n", 'bold')) - return - - self.pause(ports) - - # true means print time - return True - - - @__console - def resume_line (self, line): - '''Resume active traffic in specified ports on TRex\n''' - parser = parsing_opts.gen_parser(self, - "resume", - self.resume_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - # find the relevant ports - ports = list(set(self.get_paused_ports()).intersection(opts.ports)) - - if not ports: - self.logger.log(format_text("No ports in valid state to resume\n", 'bold')) - return - - return self.resume(ports) - - # true means print time - return True - - - @__console - def clear_stats_line (self, line): - '''Clear cached local statistics\n''' - # define a parser - parser = parsing_opts.gen_parser(self, - "clear", - self.clear_stats_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL) - - opts = parser.parse_args(line.split()) - - if opts is None: - return - - self.clear_stats(opts.ports) - - - - - @__console - def show_stats_line (self, line): - '''Fetch statistics from TRex server by port\n''' - # define a parser - parser = parsing_opts.gen_parser(self, - "stats", - self.show_stats_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL, - parsing_opts.STATS_MASK) - - opts = parser.parse_args(line.split()) - - if opts is None: - return - - # determine stats mask - mask = self.__get_mask_keys(**self.__filter_namespace_args(opts, trex_stats.ALL_STATS_OPTS)) - if not mask: - # set to show all stats if no filter was given - mask = trex_stats.ALL_STATS_OPTS - - stats_opts = trex_stats.ALL_STATS_OPTS.intersection(mask) - - stats = self._get_formatted_stats(opts.ports, mask) - - - # print stats to screen - for stat_type, stat_data in stats.iteritems(): - text_tables.print_table_with_header(stat_data.text_table, stat_type) - - - @__console - def show_streams_line(self, line): - '''Fetch streams statistics from TRex server by port\n''' - # define a parser - parser = parsing_opts.gen_parser(self, - "streams", - self.show_streams_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL, - parsing_opts.STREAMS_MASK) - - opts = parser.parse_args(line.split()) - - if opts is None: - return - - streams = self._get_streams(opts.ports, set(opts.streams)) - if not streams: - self.logger.log(format_text("No streams found with desired filter.\n", "bold", "magenta")) - - else: - # print stats to screen - for stream_hdr, port_streams_data in streams.iteritems(): - text_tables.print_table_with_header(port_streams_data.text_table, - header= stream_hdr.split(":")[0] + ":", - untouched_header= stream_hdr.split(":")[1]) - - - - - @__console - def validate_line (self, line): - '''validates port(s) stream configuration\n''' - - parser = parsing_opts.gen_parser(self, - "validate", - self.validate_line.__doc__, - parsing_opts.PORT_LIST_WITH_ALL) - - opts = parser.parse_args(line.split()) - if opts is None: - return - - self.validate(opts.ports) - - - - \ No newline at end of file diff --git a/scripts/automation/trex_control_plane/client/trex_stateless_sim.py b/scripts/automation/trex_control_plane/client/trex_stateless_sim.py deleted file mode 100644 index 1452cdd1..00000000 --- a/scripts/automation/trex_control_plane/client/trex_stateless_sim.py +++ /dev/null @@ -1,430 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -Itay Marom -Cisco Systems, Inc. - -Copyright (c) 2015-2015 Cisco Systems, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -try: - # support import for Python 2 - import outer_packages -except ImportError: - # support import for Python 3 - import client.outer_packages - -from common.trex_stl_exceptions import STLError -from yaml import YAMLError -from common.trex_streams import * -from client_utils import parsing_opts - -import re -import json - - - -import argparse -import tempfile -import subprocess -import os -from dpkt import pcap -from operator import itemgetter - -class BpSimException(Exception): - pass - -def merge_cap_files (pcap_file_list, out_filename, delete_src = False): - - out_pkts = [] - if not all([os.path.exists(f) for f in pcap_file_list]): - print "failed to merge cap file list...\nnot all files exist\n" - return - - # read all packets to a list - for src in pcap_file_list: - f = open(src, 'r') - reader = pcap.Reader(f) - pkts = reader.readpkts() - out_pkts += pkts - f.close() - if delete_src: - os.unlink(src) - - # sort by the timestamp - out_pkts = sorted(out_pkts, key=itemgetter(0)) - - - out = open(out_filename, 'w') - out_writer = pcap.Writer(out) - - for ts, pkt in out_pkts: - out_writer.writepkt(pkt, ts) - - out.close() - - - -# stateless simulation -class STLSim(object): - def __init__ (self, bp_sim_path = None, handler = 0, port_id = 0): - - if not bp_sim_path: - # auto find scripts - m = re.match(".*/trex-core", os.getcwd()) - if not m: - raise STLError('cannot find BP sim path, please provide it') - - self.bp_sim_path = os.path.join(m.group(0), 'scripts') - - else: - self.bp_sim_path = bp_sim_path - - # dummies - self.handler = handler - self.port_id = port_id - - - def load_input_file (self, input_file): - # try YAML - try: - streams_db = CStreamsDB() - stream_list = streams_db.load_yaml_file(input_file) - - # convert to new style stream object - return [HACKSTLStream(stream) for stream in stream_list.compiled] - except YAMLError: - pass - - # try python - try: - basedir = os.path.dirname(input_file) - sys.path.append(basedir) - - file = os.path.basename(input_file).split('.')[0] - module = __import__(file, globals(), locals(), [], -1) - - return module.register().get_streams() - - except (AttributeError, ImportError) as e: - print "specific error: {0}".format(e) - - raise STLError("bad format input file '{0}'".format(input_file)) - - - def generate_start_cmd (self, mult = "1", force = True, duration = -1): - return {"id":1, - "jsonrpc": "2.0", - "method": "start_traffic", - "params": {"handler": self.handler, - "force": force, - "port_id": self.port_id, - "mul": parsing_opts.decode_multiplier(mult), - "duration": duration} - } - - - - # run command - # input_list - a list of streams or YAML files - # outfile - pcap file to save output, if None its a dry run - # dp_core_count - how many DP cores to use - # dp_core_index - simulate only specific dp core without merging - # is_debug - debug or release image - # pkt_limit - how many packets to simulate - # mult - multiplier - # mode - can be 'valgrind, 'gdb', 'json' or 'none' - def run (self, - input_list, - outfile = None, - dp_core_count = 1, - dp_core_index = None, - is_debug = True, - pkt_limit = 5000, - mult = "1", - duration = -1, - mode = 'none'): - - if not mode in ['none', 'gdb', 'valgrind', 'json']: - raise STLArgumentError('mode', mode) - - # listify - input_list = input_list if isinstance(input_list, list) else [input_list] - - # check streams arguments - if not all([isinstance(i, (STLStream, str)) for i in input_list]): - raise STLArgumentError('input_list', input_list) - - # split to two type - input_files = [x for x in input_list if isinstance(x, str)] - stream_list = [x for x in input_list if isinstance(x, STLStream)] - - # handle YAMLs - for input_file in input_files: - stream_list += self.load_input_file(input_file) - - - # load streams - cmds_json = [] - for stream in stream_list: - cmd = {"id":1, - "jsonrpc": "2.0", - "method": "add_stream", - "params": {"handler": self.handler, - "port_id": self.port_id, - "stream_id": stream.get_id(), - "stream": stream.to_json()} - } - - cmds_json.append(cmd) - - # generate start command - cmds_json.append(self.generate_start_cmd(mult = mult, - force = True, - duration = duration)) - - if mode == 'json': - print json.dumps(cmds_json, indent = 4, separators=(',', ': '), sort_keys = True) - return - - # start simulation - self.outfile = outfile - self.dp_core_count = dp_core_count - self.dp_core_index = dp_core_index - self.is_debug = is_debug - self.pkt_limit = pkt_limit - self.mult = mult - self.duration = duration, - self.mode = mode - - self.__run(cmds_json) - - - # internal run - def __run (self, cmds_json): - - # write to temp file - f = tempfile.NamedTemporaryFile(delete = False) - f.write(json.dumps(cmds_json)) - f.close() - - # launch bp-sim - try: - self.execute_bp_sim(f.name) - finally: - os.unlink(f.name) - - - - def execute_bp_sim (self, json_filename): - if self.is_debug: - exe = os.path.join(self.bp_sim_path, 'bp-sim-64-debug') - else: - exe = os.path.join(self.bp_sim_path, 'bp-sim-64') - - if not os.path.exists(exe): - raise STLError("'{0}' does not exists, please build it before calling the simulation".format(exe)) - - - cmd = [exe, - '--pcap', - '--sl', - '--cores', - str(self.dp_core_count), - '--limit', - str(self.pkt_limit), - '-f', - json_filename] - - # out or dry - if not self.outfile: - cmd += ['--dry'] - cmd += ['-o', '/dev/null'] - else: - cmd += ['-o', self.outfile] - - if self.dp_core_index != None: - cmd += ['--core_index', str(self.dp_core_index)] - - if self.mode == 'valgrind': - cmd = ['valgrind', '--leak-check=full', '--error-exitcode=1'] + cmd - - elif self.mode == 'gdb': - cmd = ['/bin/gdb', '--args'] + cmd - - print "executing command: '{0}'".format(" ".join(cmd)) - rc = subprocess.call(cmd) - if rc != 0: - raise STLError('simulation has failed with error code {0}'.format(rc)) - - self.merge_results() - - - def merge_results (self): - if not self.outfile: - return - - if self.dp_core_count == 1: - return - - if self.dp_core_index != None: - return - - - print "Mering cores output to a single pcap file...\n" - inputs = ["{0}-{1}".format(self.outfile, index) for index in xrange(0, self.dp_core_count)] - merge_cap_files(inputs, self.outfile, delete_src = True) - - - - -def is_valid_file(filename): - if not os.path.isfile(filename): - raise argparse.ArgumentTypeError("The file '%s' does not exist" % filename) - - return filename - - -def unsigned_int (x): - x = int(x) - if x < 0: - raise argparse.ArgumentTypeError("argument must be >= 0") - - return x - -def setParserOptions(): - parser = argparse.ArgumentParser(prog="stl_sim.py") - - parser.add_argument("-f", - dest ="input_file", - help = "input file in YAML or Python format", - type = is_valid_file, - required=True) - - parser.add_argument("-o", - dest = "output_file", - default = None, - help = "output file in ERF format") - - - parser.add_argument("-c", "--cores", - help = "DP core count [default is 1]", - dest = "dp_core_count", - default = 1, - type = int, - choices = xrange(1, 9)) - - parser.add_argument("-n", "--core_index", - help = "Record only a specific core", - dest = "dp_core_index", - default = None, - type = int) - - parser.add_argument("-r", "--release", - help = "runs on release image instead of debug [default is False]", - action = "store_true", - default = False) - - - parser.add_argument("-l", "--limit", - help = "limit test total packet count [default is 5000]", - default = 5000, - type = unsigned_int) - - parser.add_argument('-m', '--multiplier', - help = parsing_opts.match_multiplier_help, - dest = 'mult', - default = "1", - type = parsing_opts.match_multiplier_strict) - - parser.add_argument('-d', '--duration', - help = "run duration", - dest = 'duration', - default = -1, - type = float) - - - group = parser.add_mutually_exclusive_group() - - group.add_argument("-x", "--valgrind", - help = "run under valgrind [default is False]", - action = "store_true", - default = False) - - group.add_argument("-g", "--gdb", - help = "run under GDB [default is False]", - action = "store_true", - default = False) - - group.add_argument("--json", - help = "generate JSON output only to stdout [default is False]", - action = "store_true", - default = False) - - return parser - - -def validate_args (parser, options): - - if options.dp_core_index: - if not options.dp_core_index in xrange(0, options.dp_core_count): - parser.error("DP core index valid range is 0 to {0}".format(options.dp_core_count - 1)) - - # zero is ok - no limit, but other values must be at least as the number of cores - if (options.limit != 0) and options.limit < options.dp_core_count: - parser.error("limit cannot be lower than number of DP cores") - - -def main (): - parser = setParserOptions() - options = parser.parse_args() - - validate_args(parser, options) - - - - if options.valgrind: - mode = 'valgrind' - elif options.gdb: - mode = 'gdb' - elif options.json: - mode = 'json' - else: - mode = 'none' - - try: - r = STLSim() - r.run(input_list = options.input_file, - outfile = options.output_file, - dp_core_count = options.dp_core_count, - dp_core_index = options.dp_core_index, - is_debug = (not options.release), - pkt_limit = options.limit, - mult = options.mult, - duration = options.duration, - mode = mode) - - except KeyboardInterrupt as e: - print "\n\n*** Caught Ctrl + C... Exiting...\n\n" - exit(1) - - except STLError as e: - print e - exit(1) - - exit(0) - -if __name__ == '__main__': - main() - - diff --git a/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py b/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py deleted file mode 100755 index 9c351175..00000000 --- a/scripts/automation/trex_control_plane/client_utils/jsonrpc_client.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/router/bin/python - -import external_packages -import zmq -import json -import general_utils -import re -from time import sleep -from collections import namedtuple -from common.trex_types import * - -class bcolors: - BLUE = '\033[94m' - GREEN = '\033[32m' - YELLOW = '\033[93m' - RED = '\033[31m' - MAGENTA = '\033[35m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - -# sub class to describe a batch -class BatchMessage(object): - def __init__ (self, rpc_client): - self.rpc_client = rpc_client - self.batch_list = [] - - def add (self, method_name, params={}): - - id, msg = self.rpc_client.create_jsonrpc_v2(method_name, params, encode = False) - self.batch_list.append(msg) - - def invoke(self, block = False): - if not self.rpc_client.connected: - return RC_ERR("Not connected to server") - - msg = json.dumps(self.batch_list) - - return self.rpc_client.send_raw_msg(msg) - - -# JSON RPC v2.0 client -class JsonRpcClient(object): - - def __init__ (self, default_server, default_port, logger): - self.logger = logger - self.connected = False - - # default values - self.port = default_port - self.server = default_server - self.id_gen = general_utils.random_id_gen() - - - def get_connection_details (self): - rc = {} - rc['server'] = self.server - rc['port'] = self.port - - return rc - - # pretty print for JSON - def pretty_json (self, json_str, use_colors = True): - pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True) - - if not use_colors: - return pretty_str - - try: - # int numbers - pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}\2{1}'.format(bcolors.BLUE, bcolors.ENDC), pretty_str) - # float - pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}\2{1}'.format(bcolors.MAGENTA, bcolors.ENDC), pretty_str) - # strings - - pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}\2{1}'.format(bcolors.RED, bcolors.ENDC), pretty_str) - pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(bcolors.MAGENTA, bcolors.RED), pretty_str) - except : - pass - - return pretty_str - - def verbose_msg (self, msg): - self.logger.log("\n\n[verbose] " + msg, level = self.logger.VERBOSE_HIGH) - - - # batch messages - def create_batch (self): - return BatchMessage(self) - - def create_jsonrpc_v2 (self, method_name, params = {}, encode = True): - msg = {} - msg["jsonrpc"] = "2.0" - msg["method"] = method_name - - msg["params"] = params - - msg["id"] = self.id_gen.next() - - if encode: - return id, json.dumps(msg) - else: - return id, msg - - - def invoke_rpc_method (self, method_name, params = {}): - if not self.connected: - return RC_ERR("Not connected to server") - - id, msg = self.create_jsonrpc_v2(method_name, params) - - return self.send_raw_msg(msg) - - - # low level send of string message - def send_raw_msg (self, msg): - - self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n") - - tries = 0 - while True: - try: - self.socket.send(msg) - break - except zmq.Again: - tries += 1 - if tries > 5: - self.disconnect() - return RC_ERR("*** [RPC] - Failed to send message to server") - - - tries = 0 - while True: - try: - response = self.socket.recv() - break - except zmq.Again: - tries += 1 - if tries > 5: - self.disconnect() - return RC_ERR("*** [RPC] - Failed to get server response at {0}".format(self.transport)) - - - self.verbose_msg("Server Response:\n\n" + self.pretty_json(response) + "\n") - - # decode - - # batch ? - response_json = json.loads(response) - - if isinstance(response_json, list): - rc_batch = RC() - - for single_response in response_json: - rc = self.process_single_response(single_response) - rc_batch.add(rc) - - return rc_batch - - else: - return self.process_single_response(response_json) - - - def process_single_response (self, response_json): - - if (response_json.get("jsonrpc") != "2.0"): - return RC_ERR("Malformed Response ({0})".format(str(response_json))) - - # error reported by server - if ("error" in response_json): - if "specific_err" in response_json["error"]: - return RC_ERR(response_json["error"]["specific_err"]) - else: - return RC_ERR(response_json["error"]["message"]) - - - # if no error there should be a result - if ("result" not in response_json): - return RC_ERR("Malformed Response ({0})".format(str(response_json))) - - return RC_OK(response_json["result"]) - - - - def disconnect (self): - if self.connected: - self.socket.close(linger = 0) - self.context.destroy(linger = 0) - self.connected = False - return RC_OK() - else: - return RC_ERR("Not connected to server") - - - def connect(self, server = None, port = None): - if self.connected: - self.disconnect() - - self.context = zmq.Context() - - self.server = (server if server else self.server) - self.port = (port if port else self.port) - - # Socket to talk to server - self.transport = "tcp://{0}:{1}".format(self.server, self.port) - - self.socket = self.context.socket(zmq.REQ) - try: - self.socket.connect(self.transport) - except zmq.error.ZMQError as e: - return RC_ERR("ZMQ Error: Bad server or port name: " + str(e)) - - self.socket.setsockopt(zmq.SNDTIMEO, 1000) - self.socket.setsockopt(zmq.RCVTIMEO, 1000) - - self.connected = True - - rc = self.invoke_rpc_method('ping') - if not rc: - self.connected = False - return rc - - return RC_OK() - - - def reconnect(self): - # connect using current values - return self.connect() - - if not self.connected: - return RC_ERR("Not connected to server") - - # reconnect - return self.connect(self.server, self.port) - - - def is_connected(self): - return self.connected - - def __del__(self): - self.logger.log("Shutting down RPC client\n") - if hasattr(self, "context"): - self.context.destroy(linger=0) - diff --git a/scripts/automation/trex_control_plane/client_utils/packet_builder.py b/scripts/automation/trex_control_plane/client_utils/packet_builder.py deleted file mode 100755 index f9031436..00000000 --- a/scripts/automation/trex_control_plane/client_utils/packet_builder.py +++ /dev/null @@ -1,1209 +0,0 @@ -#!/router/bin/python - -import external_packages -import dpkt -import socket -import binascii -import copy -import random -import string -import struct -import re -import itertools -from abc import ABCMeta, abstractmethod -from collections import namedtuple -import base64 - -from packet_builder_interface import CTrexPktBuilderInterface - -class CTRexPktBuilder(CTrexPktBuilderInterface): - """ - This class defines the TRex API of building a packet using dpkt package. - Using this class the user can also define how TRex will handle the packet by specifying the VM setting. - """ - def __init__(self, max_pkt_size=dpkt.ethernet.ETH_LEN_MAX): - """ - Instantiate a CTRexPktBuilder object - - :parameters: - None - - """ - super(CTRexPktBuilder, self).__init__() - self._packet = None - self._pkt_by_hdr = {} - self._pkt_top_layer = None - self._max_pkt_size = max_pkt_size - self.vm = CTRexPktBuilder.CTRexVM() - self.metadata = "" - - def clone (self): - return copy.deepcopy(self) - - def add_pkt_layer(self, layer_name, pkt_layer): - """ - This method adds additional header to the already existing packet - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l2", "l4_tcp", etc. - - pkt_layer : dpkt.Packet obj - a dpkt object, generally from higher layer, that will be added on top of existing layer. - - :raises: - + :exc:`ValueError`, in case the desired layer_name already exists. - - """ - assert isinstance(pkt_layer, dpkt.Packet) - if layer_name in self._pkt_by_hdr: - raise ValueError("Given layer name '{0}' already exists.".format(layer_name)) - else: - dup_pkt = copy.copy(pkt_layer) # using copy of layer to avoid cyclic packets that may lead to infinite loop - if not self._pkt_top_layer: # this is the first header added - self._packet = dup_pkt - else: - self._pkt_top_layer.data = dup_pkt - self._pkt_top_layer = dup_pkt - self._pkt_by_hdr[layer_name] = dup_pkt - return - - def set_ip_layer_addr(self, layer_name, attr, ip_addr, ip_type="ipv4"): - """ - This method sets the IP address fields of an IP header (source or destination, for both IPv4 and IPv6) - using a human readable addressing representation. - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l3_ip", etc. - - attr: str - a string representation of the sub-field to be set: - - + "src" for source - + "dst" for destination - - ip_addr: str - a string representation of the IP address to be set. - Example: "10.0.0.1" for IPv4, or "5001::DB8:1:3333:1:1" for IPv6 - - ip_type : str - a string representation of the IP version to be set: - - + "ipv4" for IPv4 - + "ipv6" for IPv6 - - Default: **ipv4** - - :raises: - + :exc:`ValueError`, in case the desired layer_name is not an IP layer - + :exc:`KeyError`, in case the desired layer_name does not exists. - - """ - try: - layer = self._pkt_by_hdr[layer_name.lower()] - if not (isinstance(layer, dpkt.ip.IP) or isinstance(layer, dpkt.ip6.IP6)): - raise ValueError("The specified layer '{0}' is not of IPv4/IPv6 type.".format(layer_name)) - else: - decoded_ip = CTRexPktBuilder._decode_ip_addr(ip_addr, ip_type) - setattr(layer, attr, decoded_ip) - except KeyError: - raise KeyError("Specified layer '{0}' doesn't exist on packet.".format(layer_name)) - - def set_ipv6_layer_addr(self, layer_name, attr, ip_addr): - """ - This method sets the IPv6 address fields of an IP header (source or destination) - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l3_ip", etc. - - attr: str - a string representation of the sub-field to be set: - - + "src" for source - + "dst" for destination - - ip_addr: str - a string representation of the IP address to be set. - Example: "5001::DB8:1:3333:1:1" - - :raises: - + :exc:`ValueError`, in case the desired layer_name is not an IPv6 layer - + :exc:`KeyError`, in case the desired layer_name does not exists. - - """ - self.set_ip_layer_addr(layer_name, attr, ip_addr, ip_type="ipv6") - - def set_eth_layer_addr(self, layer_name, attr, mac_addr): - """ - This method sets the ethernet address fields of an Ethernet header (source or destination) - using a human readable addressing representation. - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l2", etc. - - attr: str - a string representation of the sub-field to be set: - + "src" for source - + "dst" for destination - - mac_addr: str - a string representation of the MAC address to be set. - Example: "00:de:34:ef:2e:f4". - - :raises: - + :exc:`ValueError`, in case the desired layer_name is not an Ethernet layer - + :exc:`KeyError`, in case the desired layer_name does not exists. - - """ - try: - layer = self._pkt_by_hdr[layer_name.lower()] - if not isinstance(layer, dpkt.ethernet.Ethernet): - raise ValueError("The specified layer '{0}' is not of Ethernet type.".format(layer_name)) - else: - decoded_mac = CTRexPktBuilder._decode_mac_addr(mac_addr) - setattr(layer, attr, decoded_mac) - except KeyError: - raise KeyError("Specified layer '{0}' doesn't exist on packet.".format(layer_name)) - - def set_layer_attr(self, layer_name, attr, val, toggle_bit=False): - """ - This method enables the user to change a value of a previously defined packet layer. - This method isn't to be used to set the data attribute of a packet with payload. - Use :func:`packet_builder.CTRexPktBuilder.set_payload` instead. - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l2", "l4_tcp", etc. - - attr : str - a string representing the attribute to be changed on desired layer - - val : - value of attribute. - - toggle_bit : bool - Indicating if trying to set a specific bit of a field, such as "do not fragment" bit of IP layer. - - Default: **False** - - :raises: - + :exc:`KeyError`, in case of missing layer (the desired layer isn't part of packet) - + :exc:`ValueError`, in case invalid attribute to the specified layer. - - """ - try: - layer = self._pkt_by_hdr[layer_name.lower()] - if attr == 'data' and not isinstance(val, dpkt.Packet): - # Don't allow setting 'data' attribute - raise ValueError("Set a data attribute with object that is not dpkt.Packet is not allowed using " - "set_layer_attr method.\nUse set_payload method instead.") - if hasattr(layer, attr): - if toggle_bit: - setattr(layer, attr, val | getattr(layer, attr, 0)) - else: - setattr(layer, attr, val) - if attr == 'data': - # re-evaluate packet from the start, possible broken link between layers - self._reevaluate_packet(layer_name.lower()) - else: - raise ValueError("Given attr name '{0}' doesn't exists on specified layer ({1}).".format(layer_name, - attr)) - except KeyError: - raise KeyError("Specified layer '{0}' doesn't exist on packet.".format(layer_name)) - - def set_layer_bit_attr(self, layer_name, attr, val): - """ - This method enables the user to set the value of a field smaller that 1 Byte in size. - This method isn't used to set full-sized fields value (>= 1 byte). - Use :func:`packet_builder.CTRexPktBuilder.set_layer_attr` instead. - - :parameters: - layer_name: str - a string representing the name of the layer. - Example: "l2", "l4_tcp", etc. - - attr : str - a string representing the attribute to be set on desired layer - - val : int - value of attribute. - This value will be set "ontop" of the existing value using bitwise "OR" operation. - - .. tip:: It is very useful to use dpkt constants to define the values of these fields. - - :raises: - + :exc:`KeyError`, in case of missing layer (the desired layer isn't part of packet) - + :exc:`ValueError`, in case invalid attribute to the specified layer. - - """ - return self.set_layer_attr(layer_name, attr, val, True) - - def set_pkt_payload(self, payload): - """ - This method sets a payload to the topmost layer of the generated packet. - This method isn't to be used to set another networking layer to the packet. - Use :func:`packet_builder.CTRexPktBuilder.set_layer_attr` instead. - - - :parameters: - payload: - a payload to be added to the packet at the topmost layer. - this object cannot be of type dpkt.Packet. - - :raises: - + :exc:`AttributeError`, in case no underlying header to host the payload. - - """ - assert isinstance(payload, str) - try: - self._pkt_top_layer.data = payload - except AttributeError: - raise AttributeError("The so far built packet doesn't contain an option for payload attachment.\n" - "Make sure to set appropriate underlying header before adding payload") - - def load_packet(self, packet): - """ - This method enables the user to change a value of a previously defined packet layer. - - :parameters: - packet: dpkt.Packet obj - a dpkt object that represents a packet. - - - :raises: - + :exc:`CTRexPktBuilder.IPAddressError`, in case invalid ip type option specified. - - """ - assert isinstance(packet, dpkt.Packet) - self._packet = copy.copy(packet) - - self._pkt_by_hdr.clear() - self._pkt_top_layer = self._packet - # analyze packet to layers - tmp_layer = self._packet - while True: - if isinstance(tmp_layer, dpkt.Packet): - layer_name = self._gen_layer_name(type(tmp_layer).__name__) - self._pkt_by_hdr[layer_name] = tmp_layer - self._pkt_top_layer = tmp_layer - try: - # check existence of upper layer - tmp_layer = tmp_layer.data - except AttributeError: - # this is the most upper header - self._pkt_by_hdr['pkt_final_payload'] = tmp_layer.data - break - else: - self._pkt_by_hdr['pkt_final_payload'] = tmp_layer - break - return - - def load_packet_from_pcap(self, pcap_path): - """ - This method loads a pcap file into a parsed packet builder object. - - :parameters: - pcap_path: str - a path to a pcap file, containing a SINGLE packet. - - :raises: - + :exc:`IOError`, in case provided path doesn't exists. - - """ - with open(pcap_path, 'r') as f: - pcap = dpkt.pcap.Reader(f) - first_packet = True - for _, buf in pcap: - # this is an iterator, can't evaluate the number of files in advance - if first_packet: - self.load_packet(dpkt.ethernet.Ethernet(buf)) - else: - raise ValueError("Provided pcap file contains more than single packet.") - # arrive here ONLY if pcap contained SINGLE packet - return - - def load_from_stream_obj(self, stream_obj): - self.load_packet_from_byte_list(stream_obj['packet']['binary']) - - - def load_packet_from_byte_list(self, byte_list): - - buf = base64.b64decode(byte_list) - # thn, load it based on dpkt parsing - self.load_packet(dpkt.ethernet.Ethernet(buf)) - - def get_packet(self, get_ptr=False): - """ - This method provides access to the built packet, as an instance or as a pointer to packet itself. - - :parameters: - get_ptr : bool - indicate whether to get a reference to packet or a copy. - Use only in advanced modes - if set to true, metadata for packet is cleared, and any further modification is not guaranteed. - - default value : False - - :return: - + the current packet built by CTRexPktBuilder object. - + None if packet is empty - - """ - if get_ptr: - self._pkt_by_hdr = {} - self._pkt_top_layer = None - return self._packet - else: - return copy.copy(self._packet) - - def get_packet_length(self): - return len(self._packet) - - def get_layer(self, layer_name): - """ - This method provides access to a specific layer of the packet, as a **copy of the layer instance**. - - :parameters: - layer_name : str - the name given to desired layer - - :return: - + a copy of the desired layer of the current packet if exists. - + None if no such layer - - """ - layer = self._pkt_by_hdr.get(layer_name) - return copy.copy(layer) if layer else None - - - # VM access methods - def set_vm_ip_range(self, ip_layer_name, ip_field, - ip_start, ip_end, operation, - ip_init = None, add_value = 0, - is_big_endian=True, val_size=4, - ip_type="ipv4", add_checksum_inst=True, - split = False): - - if ip_field not in ["src", "dst"]: - raise ValueError("set_vm_ip_range only available for source ('src') or destination ('dst') ip addresses") - # set differences between IPv4 and IPv6 - if ip_type == "ipv4": - ip_class = dpkt.ip.IP - ip_addr_size = val_size if val_size <= 4 else 4 - elif ip_type == "ipv6": - ip_class = dpkt.ip6.IP6 - ip_addr_size = val_size if val_size <= 8 else 4 - else: - raise CTRexPktBuilder.IPAddressError() - - self._verify_layer_prop(ip_layer_name, ip_class) - trim_size = ip_addr_size*2 - start_val = int(binascii.hexlify(CTRexPktBuilder._decode_ip_addr(ip_start, ip_type))[-trim_size:], 16) - end_val = int(binascii.hexlify(CTRexPktBuilder._decode_ip_addr(ip_end, ip_type))[-trim_size:], 16) - - if ip_init == None: - init_val = start_val - else: - init_val = int(binascii.hexlify(CTRexPktBuilder._decode_ip_addr(ip_init, ip_type))[-trim_size:], 16) - - - # All validations are done, start adding VM instructions - flow_var_name = "{layer}__{field}".format(layer=ip_layer_name, field=ip_field) - - hdr_offset, field_abs_offset = self._calc_offset(ip_layer_name, ip_field, ip_addr_size) - self.vm.add_flow_man_inst(flow_var_name, size=ip_addr_size, operation=operation, - init_value=init_val, - min_value=start_val, - max_value=end_val) - self.vm.add_write_flow_inst(flow_var_name, field_abs_offset) - self.vm.set_vm_off_inst_field(flow_var_name, "add_value", add_value) - self.vm.set_vm_off_inst_field(flow_var_name, "is_big_endian", is_big_endian) - if ip_type == "ipv4" and add_checksum_inst: - self.vm.add_fix_checksum_inst(self._pkt_by_hdr.get(ip_layer_name), hdr_offset) - - if split: - self.vm.set_split_by_var(flow_var_name) - - - def set_vm_eth_range(self, eth_layer_name, eth_field, - mac_init, mac_start, mac_end, add_value, - operation, val_size=4, is_big_endian=False): - if eth_field not in ["src", "dst"]: - raise ValueError("set_vm_eth_range only available for source ('src') or destination ('dst') eth addresses") - self._verify_layer_prop(eth_layer_name, dpkt.ethernet.Ethernet) - eth_addr_size = val_size if val_size <= 4 else 4 - trim_size = eth_addr_size*2 - init_val = int(binascii.hexlify(CTRexPktBuilder._decode_mac_addr(mac_init))[-trim_size:], 16) - start_val = int(binascii.hexlify(CTRexPktBuilder._decode_mac_addr(mac_start))[-trim_size:], 16) - end_val = int(binascii.hexlify(CTRexPktBuilder._decode_mac_addr(mac_end))[-trim_size:], 16) - # All validations are done, start adding VM instructions - flow_var_name = "{layer}__{field}".format(layer=eth_layer_name, field=eth_field) - hdr_offset, field_abs_offset = self._calc_offset(eth_layer_name, eth_field, eth_addr_size) - self.vm.add_flow_man_inst(flow_var_name, size=8, operation=operation, - init_value=init_val, - min_value=start_val, - max_value=end_val) - self.vm.add_write_flow_inst(flow_var_name, field_abs_offset) - self.vm.set_vm_off_inst_field(flow_var_name, "add_value", add_value) - self.vm.set_vm_off_inst_field(flow_var_name, "is_big_endian", is_big_endian) - - def set_vm_custom_range(self, layer_name, hdr_field, - init_val, start_val, end_val, add_val, val_size, - operation, is_big_endian=True, range_name="", - add_checksum_inst=True): - # verify input validity for init/start/end values - for val in [init_val, start_val, end_val]: - if not isinstance(val, int): - raise ValueError("init/start/end values are expected integers, but received type '{0}'". - format(type(val))) - self._verify_layer_prop(layer_name=layer_name, field_name=hdr_field) - if not range_name: - range_name = "{layer}__{field}".format(layer=layer_name, field=hdr_field) - trim_size = val_size*2 - hdr_offset, field_abs_offset = self._calc_offset(layer_name, hdr_field, val_size) - self.vm.add_flow_man_inst(range_name, size=val_size, operation=operation, - init_value=init_val, - min_value=start_val, - max_value=end_val) - self.vm.add_write_flow_inst(range_name, field_abs_offset) - self.vm.set_vm_off_inst_field(range_name, "add_value", add_val) - self.vm.set_vm_off_inst_field(range_name, "is_big_endian", is_big_endian) - if isinstance(self._pkt_by_hdr.get(layer_name), dpkt.ip.IP) and add_checksum_inst: - self.vm.add_fix_checksum_inst(self._pkt_by_hdr.get(layer_name), hdr_offset) - - def get_vm_data(self): - return self.vm.dump() - - def compile (self): - pass - - def dump_pkt(self, encode = True): - """ - Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) - - :parameters: - encode : bool - Encode using base64. (disable for debug) - - Default: **True** - - :return: - + packet representation as array of bytes - - :raises: - + :exc:`CTRexPktBuilder.EmptyPacketError`, in case packet is empty. - - """ - if self._packet is None: - raise CTRexPktBuilder.EmptyPacketError() - - if encode: - return {"binary": base64.b64encode(str(self._packet)), - "meta": self.metadata} - return {"binary": str(self._packet), - "meta": self.metadata} - - - def dump_pkt_to_pcap(self, file_path, ts=None): - """ - Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) - - :parameters: - file_path : str - a path (including filename) to which to write to pcap file to. - - ts : int - a timestamp to attach to the packet when dumped to pcap file. - if ts in None, then time.time() is used to set the timestamp. - - Default: **None** - - :return: - None - - :raises: - + :exc:`CTRexPktBuilder.EmptyPacketError`, in case packet is empty. - - """ - if self._packet is None: - raise CTRexPktBuilder.EmptyPacketError() - try: - with open(file_path, 'wb') as f: - pcap_wr = dpkt.pcap.Writer(f) - pcap_wr.writepkt(self._packet, ts) - return - except IOError: - raise IOError(2, "The provided path could not be accessed") - - def get_packet_layers(self, depth_limit=Ellipsis): - if self._packet is None: - raise CTRexPktBuilder.EmptyPacketError() - cur_layer = self._packet - layer_types = [] - if depth_limit == Ellipsis: - iterator = itertools.count(1) - else: - iterator = xrange(depth_limit) - for _ in iterator: - # append current layer type - if isinstance(cur_layer, dpkt.Packet): - layer_types.append(type(cur_layer).__name__) - else: - # if not dpkt layer, refer as payload - layer_types.append("PLD") - # advance to next layer - if not hasattr(cur_layer, "data"): - break - else: - cur_layer = cur_layer.data - return layer_types - - def export_pkt(self, file_path, link_pcap=False, pcap_name=None, pcap_ts=None): - pass - - # ----- internal methods ----- # - def _reevaluate_packet(self, layer_name): - cur_layer = self._packet - known_layers = set(self._pkt_by_hdr.keys()) - found_layers = set() - while True: - pointing_layer_name = self._find_pointing_layer(known_layers, cur_layer) - found_layers.add(pointing_layer_name) - if self._pkt_by_hdr[layer_name] is cur_layer: - self._pkt_top_layer = cur_layer - disconnected_layers = known_layers.difference(found_layers) - # remove disconnected layers - for layer in disconnected_layers: - self._pkt_by_hdr.pop(layer) - break - else: - cur_layer = cur_layer.data - - def _gen_layer_name(self, layer_class_name): - assert isinstance(layer_class_name, str) - layer_name = layer_class_name.lower() - idx = 1 - while True: - tmp_name = "{name}_{id}".format(name=layer_name, id=idx) - if tmp_name not in self._pkt_by_hdr: - return tmp_name - else: - idx += 1 - - def _find_pointing_layer(self, known_layers, layer_obj): - assert isinstance(known_layers, set) - for layer in known_layers: - if self._pkt_by_hdr[layer] is layer_obj: - return layer - - def _calc_offset(self, layer_name, hdr_field, hdr_field_size): - pkt_header = self._pkt_by_hdr.get(layer_name) - hdr_offset = len(self._packet) - len(pkt_header) - inner_hdr_offsets = [] - for field in pkt_header.__hdr__: - if field[0] == hdr_field: - field_size = struct.calcsize(field[1]) - if field_size == hdr_field_size: - break - elif field_size < hdr_field_size: - raise CTRexPktBuilder.PacketLayerError(layer_name, - "The specified field '{0}' size is smaller than given range" - " size ('{1}')".format(hdr_field, hdr_field_size)) - else: - inner_hdr_offsets.append(field_size - hdr_field_size) - break - else: - inner_hdr_offsets.append(struct.calcsize(field[1])) - return hdr_offset, hdr_offset + sum(inner_hdr_offsets) - - def _verify_layer_prop(self, layer_name, layer_type=None, field_name=None): - if layer_name not in self._pkt_by_hdr: - raise CTRexPktBuilder.PacketLayerError(layer_name) - pkt_layer = self._pkt_by_hdr.get(layer_name) - if layer_type: - # check for layer type - if not isinstance(pkt_layer, layer_type): - raise CTRexPktBuilder.PacketLayerTypeError(layer_name, type(pkt_layer), layer_type) - if field_name and not hasattr(pkt_layer, field_name): - # check if field exists on certain header - raise CTRexPktBuilder.PacketLayerError(layer_name, "The specified field '{0}' does not exists on " - "given packet layer ('{1}')".format(field_name, - layer_name)) - return - - @property - def payload_gen(self): - return CTRexPktBuilder.CTRexPayloadGen(self._packet, self._max_pkt_size) - - @staticmethod - def _decode_mac_addr(mac_addr): - """ - Static method to test for MAC address validity. - - :parameters: - mac_addr : str - a string representing an MAC address, separated by ':' or '-'. - - examples: '00:de:34:ef:2e:f4', '00-de-34-ef-2e-f4 - - :return: - + an hex-string representation of the MAC address. - for example, ip 00:de:34:ef:2e:f4 will return '\x00\xdeU\xef.\xf4' - - :raises: - + :exc:`CTRexPktBuilder.MACAddressError`, in case invalid ip type option specified. - - """ - tmp_mac = mac_addr.lower().replace('-', ':') - if re.match("[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", tmp_mac): - return binascii.unhexlify(tmp_mac.replace(':', '')) - # another option for both Python 2 and 3: - # codecs.decode(tmp_mac.replace(':', ''), 'hex') - else: - raise CTRexPktBuilder.MACAddressError() - - @staticmethod - def _decode_ip_addr(ip_addr, ip_type): - """ - Static method to test for IPv4/IPv6 address validity. - - :parameters: - ip_addr : str - a string representing an IP address (IPv4/IPv6) - - ip_type : str - The type of IP to be checked. - Valid types: "ipv4", "ipv6". - - :return: - + an hex-string representation of the ip address. - for example, ip 1.2.3.4 will return '\x01\x02\x03\x04' - - :raises: - + :exc:`CTRexPktBuilder.IPAddressError`, in case invalid ip type option specified. - - """ - if ip_type == "ipv4": - try: - return socket.inet_pton(socket.AF_INET, ip_addr) - except AttributeError: # no inet_pton here, sorry - # try: - return socket.inet_aton(ip_addr) - # except socket.error: - # return False - # return ip_addr.count('.') == 3 - except socket.error: # not a valid address - raise CTRexPktBuilder.IPAddressError() - elif ip_type == "ipv6": - try: - return socket.inet_pton(socket.AF_INET6, ip_addr) - except socket.error: # not a valid address - raise CTRexPktBuilder.IPAddressError() - else: - raise CTRexPktBuilder.IPAddressError() - - # ------ private classes ------ # - class CTRexPayloadGen(object): - - def __init__(self, packet_ref, max_pkt_size): - self._pkt_ref = packet_ref - self._max_pkt_size = max_pkt_size - - def gen_random_str(self): - gen_length = self._calc_gen_length() - # return a string of size gen_length bytes, to pad the packet to its max_size - return ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) - for _ in range(gen_length)) - - def gen_repeat_ptrn(self, ptrn_to_repeat): - gen_length = self._calc_gen_length() - if isinstance(ptrn_to_repeat, str): - # generate repeated string - return (ptrn_to_repeat * (gen_length/len(ptrn_to_repeat) + 1))[:gen_length] - elif isinstance(ptrn_to_repeat, int): - ptrn = binascii.unhexlify(hex(ptrn_to_repeat)[2:]) - return (ptrn * (gen_length/len(ptrn) + 1))[:gen_length] - elif isinstance(ptrn_to_repeat, tuple): - if not all((isinstance(x, int) and (x < 255) and (x >= 0)) - for x in ptrn_to_repeat): - raise ValueError("All numbers in tuple must be in range 0 <= number <= 255 ") - # generate repeated sequence - to_pack = (ptrn_to_repeat * (gen_length/len(ptrn_to_repeat) + 1))[:gen_length] - return struct.pack('B'*gen_length, *to_pack) - else: - raise ValueError("Given ptrn_to_repeat argument type ({0}) is illegal.". - format(type(ptrn_to_repeat))) - - def _calc_gen_length(self): - return self._max_pkt_size - len(self._pkt_ref) - - class CTRexVM(object): - """ - This class defines the TRex VM which represents how TRex will regenerate packets. - The packets will be regenerated based on the built packet containing this class. - """ - InstStore = namedtuple('InstStore', ['type', 'inst']) - - def __init__(self): - """ - Instantiate a CTRexVM object - - :parameters: - None - """ - super(CTRexPktBuilder.CTRexVM, self).__init__() - self.vm_variables = {} - self._inst_by_offset = {} # this data structure holds only offset-related instructions, ordered in tuples - self._off_inst_by_name = {} - self.split_by_var = '' - - def set_vm_var_field(self, var_name, field_name, val, offset_inst=False): - """ - Set VM variable field. Only existing variables are allowed to be changed. - - :parameters: - var_name : str - a string representing the name of the VM variable to be changed. - field_name : str - a string representing the field name of the VM variable to be changed. - val : - a value to be applied to field_name field of the var_name VM variable. - - :raises: - + :exc:`KeyError`, in case invalid var_name has been specified. - + :exc:`CTRexPktBuilder.VMVarFieldTypeError`, in case mismatch between `val` and allowed type. - + :exc:`CTRexPktBuilder.VMVarValueError`, in case val isn't one of allowed options of field_name. - - """ - if offset_inst: - return self._off_inst_by_name[var_name].inst.set_field(field_name, val) - else: - return self.vm_variables[var_name].set_field(field_name, val) - - def set_vm_off_inst_field(self, var_name, field_name, val): - return self.set_vm_var_field(var_name, field_name, val, True) - - def add_flow_man_inst(self, name, **kwargs): - """ - Adds a new flow manipulation object to the VM instance. - - :parameters: - name : str - name of the manipulation, must be distinct. - Example: 'source_ip_change' - - **kwargs** : dict - optional, set flow_man fields on initialization (key = field_name, val = field_val). - Must be used with legit fields, see :func:`CTRexPktBuilder.CTRexVM.CTRexVMVariable.set_field`. - - :return: - None - - :raises: - + :exc:`CTRexPktBuilder.VMVarNameExistsError`, in case of desired flow_man name already taken. - + Exceptions from :func:`CTRexPktBuilder.CTRexVM.CTRexVMVariable.set_field` method. - Will rise when VM variables were misconfiguration. - """ - if name not in self.vm_variables: - self.vm_variables[name] = self.CTRexVMFlowVariable(name) - # try configuring VM instruction attributes - for (field, value) in kwargs.items(): - self.vm_variables[name].set_field(field, value) - else: - raise CTRexPktBuilder.VMVarNameExistsError(name) - - def add_fix_checksum_inst(self, linked_ipv4_obj, offset_to_obj=14, name=None): - # check if specified linked_ipv4_obj is indeed an ipv4 object - if not (isinstance(linked_ipv4_obj, dpkt.ip.IP)): - raise ValueError("The provided layer object is not of IPv4.") - if not name: - name = "checksum_{off}".format(off=offset_to_obj) # name will override previous checksum inst, OK - new_checksum_inst = self.CTRexVMChecksumInst(name, offset_to_obj) - # store the checksum inst in the end of the IP header (20 Bytes long) - inst = self.InstStore('checksum', new_checksum_inst) - self._inst_by_offset[offset_to_obj + 20] = inst - self._off_inst_by_name[name] = inst - - def add_write_flow_inst(self, name, pkt_offset, **kwargs): - if name not in self.vm_variables: - raise KeyError("Trying to add write_flow_var instruction to a not-exists VM flow variable ('{0}')". - format(name)) - else: - new_write_inst = self.CTRexVMWrtFlowVarInst(name, pkt_offset) - # try configuring VM instruction attributes - for (field, value) in kwargs.items(): - new_write_inst.set_field(field, value) - # add the instruction to the date-structure - inst = self.InstStore('write', new_write_inst) - self._inst_by_offset[pkt_offset] = inst - self._off_inst_by_name[name] = inst - - def load_flow_man(self, flow_obj): - """ - Loads an outer VM variable (instruction) into current VM. - The outer VM variable must contain different name than existing VM variables currently registered on VM. - - :parameters: - flow_obj : CTRexVMVariable - a CTRexVMVariable to be loaded into VM variable sets. - - :return: - list holds variables data of VM - - """ - assert isinstance(flow_obj, CTRexPktBuilder.CTRexVM.CTRexVMFlowVariable) - if flow_obj.name not in self.vm_variables.keys(): - self.vm_variables[flow_obj.name] = flow_obj - else: - raise CTRexPktBuilder.VMVarNameExistsError(flow_obj.name) - - def set_split_by_var (self, var_name): - if var_name not in self.vm_variables: - raise KeyError("cannot set split by var to an unknown VM var ('{0}')". - format(var_name)) - - self.split_by_var = var_name - - def dump(self): - """ - dumps a VM variables (instructions) and split_by_var into a dict data structure. - - :parameters: - None - - :return: - dict with VM instructions as list and split_by_var as str - - """ - - # at first, dump all CTRexVMFlowVariable instructions - inst_array = [var.dump() if hasattr(var, 'dump') else var - for key, var in self.vm_variables.items()] - # then, dump all the CTRexVMWrtFlowVarInst and CTRexVMChecksumInst instructions - inst_array += [self._inst_by_offset.get(key).inst.dump() - for key in sorted(self._inst_by_offset)] - return {'instructions': inst_array, 'split_by_var': self.split_by_var} - - class CVMAbstractInstruction(object): - __metaclass__ = ABCMeta - - def __init__(self, name): - """ - Instantiate a CTRexVMVariable object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CVMAbstractInstruction, self).__init__() - self.name = name - - def set_field(self, field_name, val): - if not hasattr(self, field_name): - raise CTRexPktBuilder.VMFieldNameError(field_name) - setattr(self, field_name, val) - - @abstractmethod - def dump(self): - pass - - class CTRexVMFlowVariable(CVMAbstractInstruction): - """ - This class defines a single VM variable to be used as part of CTRexVar object. - """ - VALID_SIZE = [1, 2, 4, 8] # size in Bytes - VALID_OPERATION = ["inc", "dec", "random"] - - def __init__(self, name): - """ - Instantiate a CTRexVMVariable object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CTRexVMFlowVariable, self).__init__(name) - # self.name = name - self.size = 4 - self.big_endian = True - self.operation = "inc" - # self.split_by_core = False - self.init_value = 1 - self.min_value = self.init_value - self.max_value = self.init_value - - def set_field(self, field_name, val): - """ - Set VM variable field. Only existing variables are allowed to be changed. - - :parameters: - field_name : str - a string representing the field name of the VM variable to be changed. - val : - a value to be applied to field_name field of the var_name VM variable. - - :return: - None - - :raises: - + :exc:`CTRexPktBuilder.VMVarNameError`, in case of illegal field name. - + :exc:`CTRexPktBuilder.VMVarFieldTypeError`, in case mismatch between `val` and allowed type. - + :exc:`CTRexPktBuilder.VMVarValueError`, in case val isn't one of allowed options of field_name. - - """ - if not hasattr(self, field_name): - raise CTRexPktBuilder.VMFieldNameError(field_name) - elif field_name == "size": - if type(val) != int: - raise CTRexPktBuilder.VMFieldTypeError("size", int) - elif val not in self.VALID_SIZE: - raise CTRexPktBuilder.VMFieldValueError("size", self.VALID_SIZE) - elif field_name in ["init_value", "min_value", "max_value"]: - if type(val) != int: - raise CTRexPktBuilder.VMFieldTypeError(field_name, int) - elif field_name == "operation": - if type(val) != str: - raise CTRexPktBuilder.VMFieldTypeError("operation", str) - elif val not in self.VALID_OPERATION: - raise CTRexPktBuilder.VMFieldValueError("operation", self.VALID_OPERATION) - # elif field_name == "split_by_core": - # val = bool(val) - # update field value on success - setattr(self, field_name, val) - - def dump(self): - """ - dumps a variable fields in a dictionary data structure. - - :parameters: - None - - :return: - dictionary holds variable data of VM variable - - """ - return {"type": "flow_var", # VM variable dump always refers to manipulate instruction. - "name": self.name, - "size": self.size, - "op": self.operation, - # "split_by_core": self.split_by_core, - "init_value": self.init_value, - "min_value": self.min_value, - "max_value": self.max_value} - - class CTRexVMChecksumInst(CVMAbstractInstruction): - - def __init__(self, name, offset): - """ - Instantiate a CTRexVMChecksumInst object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CTRexVMChecksumInst, self).__init__(name) - self.pkt_offset = offset - - def dump(self): - return {"type": "fix_checksum_ipv4", - "pkt_offset": int(self.pkt_offset)} - - class CTRexVMWrtFlowVarInst(CVMAbstractInstruction): - - def __init__(self, name, pkt_offset): - """ - Instantiate a CTRexVMWrtFlowVarInst object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CTRexVMWrtFlowVarInst, self).__init__(name) - self.pkt_offset = int(pkt_offset) - self.add_value = 0 - self.is_big_endian = False - - def set_field(self, field_name, val): - if not hasattr(self, field_name): - raise CTRexPktBuilder.VMFieldNameError(field_name) - elif field_name == 'pkt_offset': - raise ValueError("pkt_offset value cannot be changed") - cur_attr_type = type(getattr(self, field_name)) - if cur_attr_type == type(val): - setattr(self, field_name, val) - else: - CTRexPktBuilder.VMFieldTypeError(field_name, cur_attr_type) - - def dump(self): - return {"type": "write_flow_var", - "name": self.name, - "pkt_offset": self.pkt_offset, - "add_value": int(self.add_value), - "is_big_endian": bool(self.is_big_endian) - } - - class CTRexVMChecksumInst(CVMAbstractInstruction): - - def __init__(self, name, offset): - """ - Instantiate a CTRexVMChecksumInst object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CTRexVMChecksumInst, self).__init__(name) - self.pkt_offset = offset - - def dump(self): - return {"type": "fix_checksum_ipv4", - "pkt_offset": int(self.pkt_offset)} - - class CTRexVMWrtFlowVarInst(CVMAbstractInstruction): - - def __init__(self, name, pkt_offset): - """ - Instantiate a CTRexVMWrtFlowVarInst object - - :parameters: - name : str - a string representing the name of the VM variable. - """ - super(CTRexPktBuilder.CTRexVM.CTRexVMWrtFlowVarInst, self).__init__(name) - self.pkt_offset = int(pkt_offset) - self.add_value = 0 - self.is_big_endian = False - - def set_field(self, field_name, val): - if not hasattr(self, field_name): - raise CTRexPktBuilder.VMFieldNameError(field_name) - elif field_name == 'pkt_offset': - raise ValueError("pkt_offset value cannot be changed") - cur_attr_type = type(getattr(self, field_name)) - if cur_attr_type == type(val): - setattr(self, field_name, val) - else: - CTRexPktBuilder.VMFieldTypeError(field_name, cur_attr_type) - - def dump(self): - return {"type": "write_flow_var", - "name": self.name, - "pkt_offset": self.pkt_offset, - "add_value": int(self.add_value), - "is_big_endian": bool(self.is_big_endian) - } - - class CPacketBuildException(Exception): - """ - This is the general Packet Building error exception class. - """ - def __init__(self, code, message): - self.code = code - self.message = message - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return u"[errcode:%r] %r" % (self.code, self.message) - - class EmptyPacketError(CPacketBuildException): - """ - This exception is used to indicate an error caused by operation performed on an empty packet. - """ - def __init__(self, message=''): - self._default_message = 'Illegal operation on empty packet.' - self.message = message or self._default_message - super(CTRexPktBuilder.EmptyPacketError, self).__init__(-10, self.message) - - class IPAddressError(CPacketBuildException): - """ - This exception is used to indicate an error on the IP addressing part of the packet. - """ - def __init__(self, message=''): - self._default_message = 'Illegal type or value of IP address has been provided.' - self.message = message or self._default_message - super(CTRexPktBuilder.IPAddressError, self).__init__(-11, self.message) - - class MACAddressError(CPacketBuildException): - """ - This exception is used to indicate an error on the MAC addressing part of the packet. - """ - def __init__(self, message=''): - self._default_message = 'Illegal MAC address has been provided.' - self.message = message or self._default_message - super(CTRexPktBuilder.MACAddressError, self).__init__(-12, self.message) - - class PacketLayerError(CPacketBuildException): - """ - This exception is used to indicate an error caused by operation performed on an non-exists layer of the packet. - """ - def __init__(self, name, message=''): - self._default_message = "The given packet layer name ({0}) does not exists.".format(name) - self.message = message or self._default_message - super(CTRexPktBuilder.PacketLayerError, self).__init__(-13, self.message) - - class PacketLayerTypeError(CPacketBuildException): - """ - This exception is used to indicate an error caused by operation performed on an non-exists layer of the packet. - """ - def __init__(self, name, layer_type, ok_type, message=''): - self._default_message = "The type of packet layer {layer_name} is of type {layer_type}, " \ - "and not of the expected {allowed_type}.".format(layer_name=name, - layer_type=layer_type, - allowed_type=ok_type.__name__) - self.message = message or self._default_message - super(CTRexPktBuilder.PacketLayerTypeError, self).__init__(-13, self.message) - - class VMVarNameExistsError(CPacketBuildException): - """ - This exception is used to indicate a duplicate usage of VM variable. - """ - def __init__(self, name, message=''): - self._default_message = 'The given VM name ({0}) already exists as part of the stream.'.format(name) - self.message = message or self._default_message - super(CTRexPktBuilder.VMVarNameExistsError, self).__init__(-21, self.message) - - class VMFieldNameError(CPacketBuildException): - """ - This exception is used to indicate that an undefined VM var field name has been accessed. - """ - def __init__(self, name, message=''): - self._default_message = "The given VM field name ({0}) is not defined and isn't legal.".format(name) - self.message = message or self._default_message - super(CTRexPktBuilder.VMFieldNameError, self).__init__(-22, self.message) - - class VMFieldTypeError(CPacketBuildException): - """ - This exception is used to indicate an illegal value has type has been given to VM variable field. - """ - def __init__(self, name, ok_type, message=''): - self._default_message = "The desired value of field {field_name} is of type {field_type}, " \ - "and not of the allowed {allowed_type}.".format(field_name=name, - field_type=type(name).__name__, - allowed_type=ok_type.__name__) - self.message = message or self._default_message - super(CTRexPktBuilder.VMFieldTypeError, self).__init__(-31, self.message) - - class VMFieldValueError(CPacketBuildException): - """ - This exception is used to indicate an error an illegal value has been assigned to VM variable field. - """ - def __init__(self, name, ok_opts, message=''): - self._default_message = "The desired value of field {field_name} is illegal.\n" \ - "The only allowed options are: {allowed_opts}.".format(field_name=name, - allowed_opts=ok_opts) - self.message = message or self._default_message - super(CTRexPktBuilder.VMFieldValueError, self).__init__(-32, self.message) - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/client_utils/packet_builder_interface.py b/scripts/automation/trex_control_plane/client_utils/packet_builder_interface.py deleted file mode 100644 index b6e7c026..00000000 --- a/scripts/automation/trex_control_plane/client_utils/packet_builder_interface.py +++ /dev/null @@ -1,43 +0,0 @@ - -# base object class for a packet builder -class CTrexPktBuilderInterface(object): - - def compile (self): - """ - Compiles the packet and VM - """ - raise Exception("implement me") - - - def dump_pkt(self): - """ - Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) - - :parameters: - None - - :return: - + packet representation as array of bytes - - :raises: - + :exc:`CTRexPktBuilder.EmptyPacketError`, in case packet is empty. - - """ - - raise Exception("implement me") - - - def get_vm_data(self): - """ - Dumps the instructions - - :parameters: - None - - :return: - + json object of instructions - - """ - - raise Exception("implement me") - diff --git a/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py b/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py deleted file mode 100644 index b1b181c6..00000000 --- a/scripts/automation/trex_control_plane/client_utils/scapy_packet_builder.py +++ /dev/null @@ -1,748 +0,0 @@ -import external_packages -import random -import string -import struct -import socket -import json -import yaml -import binascii -import base64 - -from packet_builder_interface import CTrexPktBuilderInterface - -from scapy.all import * - - - -class CTRexPacketBuildException(Exception): - """ - This is the general Packet Building error exception class. - """ - def __init__(self, code, message): - self.code = code - self.message = message - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return u"[errcode:%r] %r" % (self.code, self.message) - -################################################################################################ - -def ipv4_str_to_num (ipv4_buffer): - - assert type(ipv4_buffer)==str, 'type of ipv4_buffer is not str' - assert len(ipv4_buffer)==4, 'size of ipv4_buffer is not 4' - res=0 - shift=24 - for i in ipv4_buffer: - res = res + (ord(i)< max_value: - raise CTRexPacketBuildException(-12, 'min is greater than max'); - if min_value == max_value: - raise CTRexPacketBuildException(-13, "min value is equal to max value, you can't use this type of range"); - - -class CTRexScIpv4SimpleRange(CTRexScFieldRangeBase): - """ - range of ipv4 ip - """ - def __init__(self, field_name, field_type, min_ip, max_ip): - super(CTRexScIpv4SimpleRange, self).__init__(field_name,field_type) - self.min_ip = min_ip - self.max_ip = max_ip - mmin=ipv4_str_to_num (is_valid_ipv4(min_ip)) - mmax=ipv4_str_to_num (is_valid_ipv4(max_ip)) - if mmin > mmax : - raise CTRexPacketBuildException(-11, 'CTRexScIpv4SimpleRange m_min ip is bigger than max'); - - -class CTRexScIpv4TupleGen(CTRexScriptsBase): - """ - range tuple - """ - FLAGS_ULIMIT_FLOWS =1 - - def __init__(self, min_ipv4, max_ipv4, num_flows=100000, min_port=1025, max_port=65535, flags=0): - super(CTRexScIpv4TupleGen, self).__init__() - self.min_ip = min_ipv4 - self.max_ip = max_ipv4 - mmin=ipv4_str_to_num (is_valid_ipv4(min_ipv4)) - mmax=ipv4_str_to_num (is_valid_ipv4(max_ipv4)) - if mmin > mmax : - raise CTRexPacketBuildException(-11, 'CTRexScIpv4SimpleRange m_min ip is bigger than max'); - - self.num_flows=num_flows; - - self.min_port =min_port - self.max_port =max_port - self.flags = flags - - -class CTRexScTrimPacketSize(CTRexScriptsBase): - """ - trim packet size. field type is CTRexScFieldRangeBase.FILED_TYPES = ["inc","dec","rand"] - """ - def __init__(self,field_type="rand",min_pkt_size=None, max_pkt_size=None): - super(CTRexScTrimPacketSize, self).__init__() - self.field_type = field_type - self.min_pkt_size = min_pkt_size - self.max_pkt_size = max_pkt_size - if max_pkt_size != None and min_pkt_size !=None : - if min_pkt_size == max_pkt_size: - raise CTRexPacketBuildException(-11, 'CTRexScTrimPacketSize min_pkt_size is the same as max_pkt_size '); - - if min_pkt_size > max_pkt_size: - raise CTRexPacketBuildException(-11, 'CTRexScTrimPacketSize min_pkt_size is bigger than max_pkt_size '); - -class CTRexScRaw(CTRexScriptsBase): - """ - raw instructions - """ - def __init__(self,list_of_commands=None): - super(CTRexScRaw, self).__init__() - if list_of_commands==None: - self.commands =[] - else: - self.commands = list_of_commands - - def add_cmd (self,cmd): - self.commands.append(cmd) - - - -################################################################################################ -# VM raw instructions -################################################################################################ - -class CTRexVmInsBase(object): - """ - instruction base - """ - def __init__(self, ins_type): - self.type = ins_type - assert type(ins_type)==str, 'type of ins_type is not str' - -class CTRexVmInsFixIpv4(CTRexVmInsBase): - def __init__(self, offset): - super(CTRexVmInsFixIpv4, self).__init__("fix_checksum_ipv4") - self.pkt_offset = offset - assert type(offset)==int, 'type of offset is not int' - - -class CTRexVmInsFlowVar(CTRexVmInsBase): - #TBD add more validation tests - - OPERATIONS =['inc', 'dec', 'random'] - VALID_SIZES =[1, 2, 4, 8] - - def __init__(self, fv_name, size, op, init_value, min_value, max_value): - super(CTRexVmInsFlowVar, self).__init__("flow_var") - self.name = fv_name; - assert type(fv_name)==str, 'type of fv_name is not str' - self.size = size - self.op = op - self.init_value = init_value - assert type(init_value)==int, 'type of init_value is not int' - self.min_value=min_value - assert type(min_value)==int, 'type of min_value is not int' - self.max_value=max_value - assert type(max_value)==int, 'type of min_value is not int' - -class CTRexVmInsWrFlowVar(CTRexVmInsBase): - def __init__(self, fv_name, pkt_offset, add_value=0, is_big_endian=True): - super(CTRexVmInsWrFlowVar, self).__init__("write_flow_var") - self.name = fv_name - assert type(fv_name)==str, 'type of fv_name is not str' - self.pkt_offset = pkt_offset - assert type(pkt_offset)==int, 'type of pkt_offset is not int' - self.add_value = add_value - assert type(add_value)==int, 'type of add_value is not int' - self.is_big_endian = is_big_endian - assert type(is_big_endian)==bool, 'type of is_big_endian is not bool' - -class CTRexVmInsTrimPktSize(CTRexVmInsBase): - def __init__(self,fv_name): - super(CTRexVmInsTrimPktSize, self).__init__("trim_pkt_size") - self.name = fv_name - assert type(fv_name)==str, 'type of fv_name is not str' - -class CTRexVmInsTupleGen(CTRexVmInsBase): - def __init__(self, fv_name, ip_min, ip_max, port_min, port_max, limit_flows, flags=0): - super(CTRexVmInsTupleGen, self).__init__("tuple_flow_var") - self.name =fv_name - assert type(fv_name)==str, 'type of fv_name is not str' - self.ip_min = ip_min; - self.ip_max = ip_max; - self.port_min = port_min; - self.port_max = port_max; - self.limit_flows = limit_flows; - self.flags =flags; - - -################################################################################################ -# -class CTRexVmEngine(object): - - def __init__(self): - """ - inlcude list of instruction - """ - super(CTRexVmEngine, self).__init__() - self.ins=[] - self.split_by_var = '' - - # return as json - def get_json (self): - inst_array = []; - # dump it as dict - for obj in self.ins: - inst_array.append(obj.__dict__); - - return {'instructions': inst_array, 'split_by_var': self.split_by_var} - - def add_ins (self,ins): - #assert issubclass(ins, CTRexVmInsBase) - self.ins.append(ins); - - def dump (self): - cnt=0; - for obj in self.ins: - print "ins",cnt - cnt = cnt +1 - print obj.__dict__ - - def dump_bjson (self): - print json.dumps(self.get_json(), sort_keys=True, indent=4) - - def dump_as_yaml (self): - print yaml.dump(self.get_json(), default_flow_style=False) - - - -################################################################################################ - -class CTRexScapyPktUtl(object): - - def __init__(self, scapy_pkt): - self.pkt = scapy_pkt - - def pkt_iter (self): - p=self.pkt; - while True: - yield p - p=p.payload - if p ==None or isinstance(p,NoPayload): - break; - - def get_list_iter(self): - l=list(self.pkt_iter()) - return l - - - def get_pkt_layers(self): - """ - return string 'IP:UDP:TCP' - """ - l=self.get_list_iter (); - l1=map(lambda p: p.name,l ); - return ":".join(l1); - - def _layer_offset(self, name, cnt = 0): - """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 - """ - save_cnt=cnt - for pkt in self.pkt_iter (): - if pkt.name == name: - if cnt==0: - return (pkt, pkt.offset) - else: - cnt=cnt -1 - - raise CTRexPacketBuildException(-11,("no layer %s-%d" % (name, save_cnt))); - - - def layer_offset(self, name, cnt = 0): - """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 - """ - save_cnt=cnt - for pkt in self.pkt_iter (): - if pkt.name == name: - if cnt==0: - return pkt.offset - else: - cnt=cnt -1 - - raise CTRexPacketBuildException(-11,("no layer %s-%d" % (name, save_cnt))); - - def get_field_offet(self, layer, layer_cnt, field_name): - """ - return offset of layer e.g 'IP',1 will return offfset of layer ip:1 - """ - t=self._layer_offset(layer,layer_cnt); - l_offset=t[1]; - layer_pkt=t[0] - - #layer_pkt.dump_fields_offsets () - - for f in layer_pkt.fields_desc: - if f.name == field_name: - return (l_offset+f.offset,f.get_size_bytes ()); - - raise CTRexPacketBuildException(-11, "no layer %s-%d." % (name, save_cnt, field_name)); - - def get_layer_offet_by_str(self, layer_des): - """ - return layer offset by string - - :parameters: - - IP:0 - IP:1 - return offset - - - """ - l1=layer_des.split(":") - layer="" - layer_cnt=0; - - if len(l1)==1: - layer=l1[0]; - else: - layer=l1[0]; - layer_cnt=int(l1[1]); - - return self.layer_offset(layer, layer_cnt) - - - - def get_field_offet_by_str(self, field_des): - """ - return field_des (offset,size) layer:cnt.field - for example - 802|1Q.vlan get 802.1Q->valn replace | with . - IP.src - IP:0.src (first IP.src like IP.src) - for example IP:1.src for internal IP - - return (offset, size) as tuple - - - """ - - s=field_des.split("."); - if len(s)!=2: - raise CTRexPacketBuildException(-11, ("field desription should be layer:cnt.field e.g IP.src or IP:1.src")); - - - layer_ex = s[0].replace("|",".") - field = s[1] - - l1=layer_ex.split(":") - layer="" - layer_cnt=0; - - if len(l1)==1: - layer=l1[0]; - else: - layer=l1[0]; - layer_cnt=int(l1[1]); - - return self.get_field_offet(layer,layer_cnt,field) - - def has_IPv4 (self): - return self.pkt.has_layer("IP"); - - def has_IPv6 (self): - return self.pkt.has_layer("IPv6"); - - def has_UDP (self): - return self.pkt.has_layer("UDP"); - -################################################################################################ - -class CTRexVmDescBase(object): - """ - instruction base - """ - def __init__(self): - pass; - - def get_obj(self): - return self; - - def get_json(self): - return self.get_obj().__dict__ - - def dump_bjson(self): - print json.dumps(self.get_json(), sort_keys=True, indent=4) - - def dump_as_yaml(self): - print yaml.dump(self.get_json(), default_flow_style=False) - - - def get_var_ref (self): - ''' - virtual function return a ref var name - ''' - return None - - def get_var_name(self): - ''' - virtual function return the varible name if exists - ''' - return None - - def compile(self,parent): - ''' - virtual function to take parent than has function name_to_offset - ''' - pass; - - -def valid_fv_size (size): - if not (size in CTRexVmInsFlowVar.VALID_SIZES): - raise CTRexPacketBuildException(-11,("flow var has not valid size %d ") % size ); - -def valid_fv_ops (op): - if not (op in CTRexVmInsFlowVar.OPERATIONS): - raise CTRexPacketBuildException(-11,("flow var does not have a valid op %s ") % op ); - -def convert_val (val): - if type(val) == int: - return val - else: - if type(val) == str: - return ipv4_str_to_num (is_valid_ipv4(val)) - else: - raise CTRexPacketBuildException(-11,("init val not valid %s ") % val ); - -def check_for_int (val): - assert type(val)==int, 'type of vcal is not int' - - -class CTRexVmDescFlowVar(CTRexVmDescBase): - def __init__(self, name, init_value=None, min_value=0, max_value=255, size=4, op="inc"): - super(CTRexVmDescFlowVar, self).__init__() - self.name = name; - assert type(name)==str, 'type of name is not str' - self.size =size - valid_fv_size(size) - self.op =op - valid_fv_ops (op) - - # choose default value for init val - if init_value == None: - init_value = max_value if op == "dec" else min_value - - self.init_value = convert_val (init_value) - self.min_value = convert_val (min_value); - self.max_value = convert_val (max_value) - - if self.min_value > self.max_value : - raise CTRexPacketBuildException(-11,("max %d is lower than min %d ") % (self.max_value,self.min_value) ); - - def get_obj (self): - return CTRexVmInsFlowVar(self.name,self.size,self.op,self.init_value,self.min_value,self.max_value); - - def get_var_name(self): - return [self.name] - - -class CTRexVmDescFixIpv4(CTRexVmDescBase): - def __init__(self, offset): - super(CTRexVmDescFixIpv4, self).__init__() - self.offset = offset; # could be a name of offset - - def get_obj (self): - return CTRexVmInsFixIpv4(self.offset); - - def compile(self,parent): - if type(self.offset)==str: - self.offset = parent._pkt_layer_offset(self.offset); - -class CTRexVmDescWrFlowVar(CTRexVmDescBase): - def __init__(self, fv_name, pkt_offset, offset_fixup=0, add_val=0, is_big=True): - super(CTRexVmDescWrFlowVar, self).__init__() - self.name =fv_name - assert type(fv_name)==str, 'type of fv_name is not str' - self.offset_fixup =offset_fixup - assert type(offset_fixup)==int, 'type of offset_fixup is not int' - self.pkt_offset =pkt_offset - self.add_val =add_val - assert type(add_val)==int,'type of add_val is not int' - self.is_big =is_big; - assert type(is_big)==bool,'type of is_big_endian is not bool' - - def get_var_ref (self): - return self.name - - def get_obj (self): - return CTRexVmInsWrFlowVar(self.name,self.pkt_offset+self.offset_fixup,self.add_val,self.is_big) - - def compile(self,parent): - if type(self.pkt_offset)==str: - t=parent._name_to_offset(self.pkt_offset) - self.pkt_offset = t[0] - - -class CTRexVmDescTrimPktSize(CTRexVmDescBase): - def __init__(self,fv_name): - super(CTRexVmDescTrimPktSize, self).__init__() - self.name = fv_name - assert type(fv_name)==str, 'type of fv_name is not str' - - def get_var_ref (self): - return self.name - - def get_obj (self): - return CTRexVmInsTrimPktSize(self.name) - - - -class CTRexVmDescTupleGen(CTRexVmDescBase): - def __init__(self,name, ip_min="0.0.0.1", ip_max="0.0.0.10", port_min=1025, port_max=65535, limit_flows=100000, flags=0): - super(CTRexVmDescTupleGen, self).__init__() - self.name = name - assert type(name)==str, 'type of fv_name is not str' - self.ip_min = convert_val(ip_min); - self.ip_max = convert_val(ip_max); - self.port_min = port_min; - check_for_int (port_min) - self.port_max = port_max; - check_for_int(port_max) - self.limit_flows = limit_flows; - check_for_int(limit_flows) - self.flags =flags; - check_for_int(flags) - - def get_var_name(self): - return [self.name+".ip",self.name+".port"] - - def get_obj (self): - return CTRexVmInsTupleGen(self.name, self.ip_min, self.ip_max, self.port_min, self.port_max, self.limit_flows, self.flags); - - -################################################################################################ - - -class CScapyTRexPktBuilder(CTrexPktBuilderInterface): - - """ - This class defines the TRex API of building a packet using dpkt package. - Using this class the user can also define how TRex will handle the packet by specifying the VM setting. - """ - def __init__(self, pkt = None, vm = None): - """ - Instantiate a CTRexPktBuilder object - - :parameters: - None - - """ - super(CScapyTRexPktBuilder, self).__init__() - - self.pkt = None - self.vm_scripts = [] # list of high level instructions - self.vm_low_level = None - self.metadata="" - - - # process packet - if pkt != None: - if not isinstance(pkt, Packet): - raise CTRexPacketBuildException(-14, "bad value for variable pkt") - self.set_packet(pkt) - - # process VM - if vm != None: - if not isinstance(vm, (CTRexScRaw, list)): - raise CTRexPacketBuildException(-14, "bad value for variable vm") - - self.add_command(vm if isinstance(vm, CTRexScRaw) else CTRexScRaw(vm)) - - - def dump_vm_data_as_yaml(self): - print yaml.dump(self.get_vm_data(), default_flow_style=False) - - def get_vm_data(self): - """ - Dumps the instructions - - :parameters: - None - - :return: - + json object of instructions - - :raises: - + :exc:`AssertionError`, in case VM is not compiled (is None). - """ - - assert self.vm_low_level is not None, 'vm_low_level is None, please use compile()' - - return self.vm_low_level.get_json() - - def dump_pkt(self, encode = True): - """ - Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) - - :parameters: - encode : bool - Encode using base64. (disable for debug) - - Default: **True** - - :return: - + packet representation as array of bytes - - :raises: - + :exc:`AssertionError`, in case packet is empty. - - """ - - assert self.pkt, 'empty packet' - - return {'binary': base64.b64encode(str(self.pkt)) if encode else str(self.pkt), - 'meta': self.metadata} - - def dump_pkt_to_pcap(self, file_path): - wrpcap(file_path, self.pkt) - - def add_command (self, script): - self.vm_scripts.append(script.clone()); - - def dump_scripts (self): - self.vm_low_level.dump_as_yaml() - - def set_packet (self, pkt): - """ - Scapy packet Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/"A"*10 - """ - self.pkt = pkt; - - - def compile (self): - self.vm_low_level=CTRexVmEngine() - assert self.pkt, 'empty packet' - self.pkt.build(); - - - for sc in self.vm_scripts: - if isinstance(sc, CTRexScRaw): - self._compile_raw(sc) - - #for obj in self.vm_scripts: - # # tuple gen script - # if isinstance(obj, CTRexScIpv4TupleGen) - # self._add_tuple_gen(tuple_gen) - - #################################################### - # private - - def _compile_raw (self,obj): - - # make sure we have varibles once - vars={}; - - # add it add var to dit - for desc in obj.commands: - var_names = desc.get_var_name() - - if var_names : - for var_name in var_names: - if vars.has_key(var_name): - raise CTRexPacketBuildException(-11,("variable %s define twice ") % (var_name) ); - else: - vars[var_name]=1 - - # check that all write exits - for desc in obj.commands: - var_name = desc.get_var_ref() - if var_name : - if not vars.has_key(var_name): - raise CTRexPacketBuildException(-11,("variable %s does not exists ") % (var_name) ); - desc.compile(self); - - for desc in obj.commands: - self.vm_low_level.add_ins(desc.get_obj()); - - - def _pkt_layer_offset (self,layer_name): - assert self.pkt != None, 'empty packet' - p_utl=CTRexScapyPktUtl(self.pkt); - return p_utl.get_layer_offet_by_str(layer_name) - - def _name_to_offset(self,field_name): - assert self.pkt != None, 'empty packet' - p_utl=CTRexScapyPktUtl(self.pkt); - return p_utl.get_field_offet_by_str(field_name) - - def _add_tuple_gen(self,tuple_gen): - - pass; - - - - diff --git a/scripts/automation/trex_control_plane/client_utils/text_tables.py b/scripts/automation/trex_control_plane/client_utils/text_tables.py index d8928da8..6b52a4a9 100644 --- a/scripts/automation/trex_control_plane/client_utils/text_tables.py +++ b/scripts/automation/trex_control_plane/client_utils/text_tables.py @@ -1,7 +1,5 @@ - -import external_packages from texttable import Texttable -from common.text_opts import format_text +from trex_control_plane.common.text_opts import format_text class TRexTextTable(Texttable): diff --git a/scripts/automation/trex_control_plane/client_utils/yaml_utils.py b/scripts/automation/trex_control_plane/client_utils/yaml_utils.py index 825d6fc9..776a51a7 100755 --- a/scripts/automation/trex_control_plane/client_utils/yaml_utils.py +++ b/scripts/automation/trex_control_plane/client_utils/yaml_utils.py @@ -16,7 +16,6 @@ limitations under the License. """ import traceback import sys -import external_packages import yaml diff --git a/scripts/automation/trex_control_plane/common/rpc_defaults.yaml b/scripts/automation/trex_control_plane/common/rpc_defaults.yaml deleted file mode 100755 index 9325a0e4..00000000 --- a/scripts/automation/trex_control_plane/common/rpc_defaults.yaml +++ /dev/null @@ -1,124 +0,0 @@ -############################################################## -#### TRex RPC stream list default values #### -############################################################## - -# this document is based on TRex RPC server spec and its fields: -# http://trex-tgn.cisco.com/trex/doc/trex_rpc_server_spec.html - -### HOW TO READ THIS FILE -# 1. Each key represents an object type -# 2. Each value can be either a value field or another object -# 2.1. If a value field, read as: -# + type: type of field -# + has_default: if the value has any default -# + default: the default value (Only appears if has_default field is 'YES') -# 2.2. If an object type, jump to corresponding object key. -# 3. If an object has more than one instance type, another layer with the type shall be added. -# For example, 'mode' object has 3 types: 'continuous', 'single_burst', 'multi_burst' -# So, 3 mode objects will be defined, named: -# - mode['continuous'] -# - mode['single_burst'] -# - mode['multi_burst'] -# In this case, there's no default for the 'type' field on the object -# 4. Some values has 'multiply' property attached. -# In such case, the loaded value will be multiplied by the multiplier -# For example, if the mode's 'pps' field value is 10, and its multiplier is 5, -# the loaded pps value will be 10*5=50 -# 5. Any object type must be listed by the user, even if all its field are defaults. -# The most basic option would be to declare the object with "[]", which stands for empty object in YAML syntax. - - -stream: - enabled: - type: boolean - has_default: YES - default: True - self_start: - type: boolean - has_default: YES - default: True - isg: - type: [int, double, string] - has_default: YES - default: 0.0 - next_stream_id: - type: string # string to allow naming binding - has_default: YES - default: -1 # no next streams - packet: - type: object - mode: - type: object - vm: - type: object - rx_stats: - type: object - -packet: - binary: - type: [array,string] - has_default: NO - meta: - type: string - has_default: YES - default: "" - -mode: - continuous: - pps: - type: [int, double] - has_default: NO - multiply: YES - single_burst: - pps: - type: [int, double] - has_default: NO - multiply: YES - total_pkts: - type: int - has_default: NO - multi_burst: - pps: - type: [int, double] - has_default: NO - multiply: YES - pkts_per_burst: - type: int - has_default: NO - ibg: - type: [int, double, string] - has_default: YES - default: 100.0 - count: - type: int - has_default: YES - default: 0 # loop forever - -rx_stats: - enabled: - type: boolean - has_default: YES - default: False - stream_id: - type: string - has_default: YES - default: False # use related stream_id - seq_enabled: - type: boolean - has_default: YES - default: False - latency_enabled: - type: boolean - has_default: YES - default: False - -vm: - instructions: - type: array - has_default: YES - default: [] - split_by_var: - type: string - has_default: YES - default: "" - diff --git a/scripts/automation/trex_control_plane/common/trex_stats.py b/scripts/automation/trex_control_plane/common/trex_stats.py deleted file mode 100755 index 3bd6e0cd..00000000 --- a/scripts/automation/trex_control_plane/common/trex_stats.py +++ /dev/null @@ -1,578 +0,0 @@ -#!/router/bin/python -from collections import namedtuple, OrderedDict, deque -from client_utils import text_tables -from common.text_opts import format_text, format_threshold, format_num -from client.trex_async_client import CTRexAsyncStats -import copy -import datetime -import time -import re -import math -import copy - -GLOBAL_STATS = 'g' -PORT_STATS = 'p' -PORT_STATUS = 'ps' -ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS} -COMPACT = {GLOBAL_STATS, PORT_STATS} - -ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) - -# use to calculate diffs relative to the previous values -# for example, BW -def calculate_diff (samples): - total = 0.0 - - weight_step = 1.0 / sum(xrange(0, len(samples))) - weight = weight_step - - for i in xrange(0, len(samples) - 1): - current = samples[i] if samples[i] > 0 else 1 - next = samples[i + 1] if samples[i + 1] > 0 else 1 - - s = 100 * ((float(next) / current) - 1.0) - - # block change by 100% - total += (min(s, 100) * weight) - weight += weight_step - - return total - - -# calculate by absolute values and not relatives (useful for CPU usage in % and etc.) -def calculate_diff_raw (samples): - total = 0.0 - - weight_step = 1.0 / sum(xrange(0, len(samples))) - weight = weight_step - - for i in xrange(0, len(samples) - 1): - current = samples[i] - next = samples[i + 1] - - total += ( (next - current) * weight ) - weight += weight_step - - return total - - -class CTRexInfoGenerator(object): - """ - This object is responsible of generating stats and information from objects maintained at - STLClient and the ports. - """ - - def __init__(self, global_stats_ref, ports_dict_ref): - self._global_stats = global_stats_ref - self._ports_dict = ports_dict_ref - - def generate_single_statistic(self, port_id_list, statistic_type): - if statistic_type == GLOBAL_STATS: - return self._generate_global_stats() - elif statistic_type == PORT_STATS: - return self._generate_port_stats(port_id_list) - pass - elif statistic_type == PORT_STATUS: - return self._generate_port_status(port_id_list) - else: - # ignore by returning empty object - return {} - - def generate_streams_info(self, port_id_list, stream_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_data = OrderedDict() - - for port_obj in relevant_ports: - streams_data = self._generate_single_port_streams_info(port_obj, stream_id_list) - if not streams_data: - continue - hdr_key = "Port {port}: {yaml_file}".format(port= port_obj.port_id, - yaml_file= streams_data.raw_data.get('referring_file', '')) - - # TODO: test for other ports with same stream structure, and join them - return_data[hdr_key] = streams_data - - return return_data - - def _generate_global_stats(self): - stats_data = self._global_stats.generate_stats() - - # build table representation - stats_table = text_tables.TRexTextInfo() - stats_table.set_cols_align(["l", "l"]) - - stats_table.add_rows([[k.replace("_", " ").title(), v] - for k, v in stats_data.iteritems()], - header=False) - - return {"global_statistics": ExportableStats(stats_data, stats_table)} - - def _generate_port_stats(self, port_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_stats_data = {} - per_field_stats = OrderedDict([("owner", []), - ("state", []), - ("--", []), - ("Tx bps", []), - ("Tx pps", []), - - ("---", []), - ("Rx bps", []), - ("Rx pps", []), - - ("----", []), - ("opackets", []), - ("ipackets", []), - ("obytes", []), - ("ibytes", []), - ("tx-bytes", []), - ("rx-bytes", []), - ("tx-pkts", []), - ("rx-pkts", []), - - ("-----", []), - ("oerrors", []), - ("ierrors", []), - - ] - ) - - total_stats = CPortStats(None) - - for port_obj in relevant_ports: - # fetch port data - port_stats = port_obj.generate_port_stats() - - total_stats += port_obj.port_stats - - # populate to data structures - return_stats_data[port_obj.port_id] = port_stats - self.__update_per_field_dict(port_stats, per_field_stats) - - total_cols = len(relevant_ports) - header = ["port"] + [port.port_id for port in relevant_ports] - - if (total_cols > 1): - self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) - header += ['total'] - total_cols += 1 - - stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["r"] * total_cols) - stats_table.set_cols_width([10] + [17] * total_cols) - stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) - - stats_table.add_rows([[k] + v - for k, v in per_field_stats.iteritems()], - header=False) - - stats_table.header(header) - - return {"port_statistics": ExportableStats(return_stats_data, stats_table)} - - def _generate_port_status(self, port_id_list): - relevant_ports = self.__get_relevant_ports(port_id_list) - - return_stats_data = {} - per_field_status = OrderedDict([("type", []), - ("maximum", []), - ("status", []) - ] - ) - - for port_obj in relevant_ports: - # fetch port data - # port_stats = self._async_stats.get_port_stats(port_obj.port_id) - port_status = port_obj.generate_port_status() - - # populate to data structures - return_stats_data[port_obj.port_id] = port_status - - self.__update_per_field_dict(port_status, per_field_status) - - stats_table = text_tables.TRexTextTable() - stats_table.set_cols_align(["l"] + ["c"]*len(relevant_ports)) - stats_table.set_cols_width([10] + [20] * len(relevant_ports)) - - stats_table.add_rows([[k] + v - for k, v in per_field_status.iteritems()], - header=False) - stats_table.header(["port"] + [port.port_id - for port in relevant_ports]) - - return {"port_status": ExportableStats(return_stats_data, stats_table)} - - def _generate_single_port_streams_info(self, port_obj, stream_id_list): - - return_streams_data = port_obj.generate_loaded_streams_sum(stream_id_list) - - if not return_streams_data.get("streams"): - # we got no streams available - return None - - # FORMAT VALUES ON DEMAND - - # because we mutate this - deep copy before - return_streams_data = copy.deepcopy(return_streams_data) - - for stream_id, stream_id_sum in return_streams_data['streams'].iteritems(): - stream_id_sum['rate_pps'] = format_num(stream_id_sum['rate_pps'], suffix='pps') - stream_id_sum['packet_type'] = self._trim_packet_headers(stream_id_sum['packet_type'], 20) - - info_table = text_tables.TRexTextTable() - info_table.set_cols_align(["c"] + ["l"] + ["r"] + ["c"] + ["r"] + ["c"]) - info_table.set_cols_width([10] + [20] + [8] + [16] + [10] + [12]) - info_table.set_cols_dtype(["t"] + ["t"] + ["t"] + ["t"] + ["t"] + ["t"]) - - info_table.add_rows([v.values() - for k, v in return_streams_data['streams'].iteritems()], - header=False) - info_table.header(["ID", "packet type", "length", "mode", "rate", "next stream"]) - - return ExportableStats(return_streams_data, info_table) - - - def __get_relevant_ports(self, port_id_list): - # fetch owned ports - ports = [port_obj - for _, port_obj in self._ports_dict.iteritems() - if port_obj.port_id in port_id_list] - - # display only the first FOUR options, by design - if len(ports) > 4: - print format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta') - ports = ports[:4] - return ports - - def __update_per_field_dict(self, dict_src_data, dict_dest_ref): - for key, val in dict_src_data.iteritems(): - if key in dict_dest_ref: - dict_dest_ref[key].append(val) - - @staticmethod - def _trim_packet_headers(headers_str, trim_limit): - if len(headers_str) < trim_limit: - # do nothing - return headers_str - else: - return (headers_str[:trim_limit-3] + "...") - - - -class CTRexStats(object): - """ This is an abstract class to represent a stats object """ - - def __init__(self): - self.reference_stats = {} - self.latest_stats = {} - self.last_update_ts = time.time() - self.history = deque(maxlen = 10) - - def __getitem__(self, item): - # override this to allow quick and clean access to fields - if not item in self.latest_stats: - return "N/A" - - # item must exist - m = re.search('_(([a-z])ps)$', item) - if m: - # this is a non-relative item - unit = m.group(2) - if unit == "b": - return self.get(item, format=True, suffix="b/sec") - elif unit == "p": - return self.get(item, format=True, suffix="pkt/sec") - else: - return self.get(item, format=True, suffix=m.group(1)) - - m = re.search('^[i|o](a-z+)$', item) - if m: - # this is a non-relative item - type = m.group(1) - if type == "bytes": - return self.get_rel(item, format=True, suffix="B") - elif type == "packets": - return self.get_rel(item, format=True, suffix="pkts") - else: - # do not format with suffix - return self.get_rel(item, format=True) - - # can't match to any known pattern, return N/A - return "N/A" - - - def generate_stats(self): - # must be implemented by designated classes (such as port/ global stats) - raise NotImplementedError() - - def update(self, snapshot): - # update - self.latest_stats = snapshot - self.history.append(snapshot) - - diff_time = time.time() - self.last_update_ts - - # 3 seconds is too much - this is the new reference - if (not self.reference_stats) or (diff_time > 3): - self.reference_stats = self.latest_stats - - self.last_update_ts = time.time() - - - def clear_stats(self): - self.reference_stats = self.latest_stats - - - def invalidate (self): - self.latest_stats = {} - - def get(self, field, format=False, suffix=""): - if not field in self.latest_stats: - return "N/A" - if not format: - return self.latest_stats[field] - else: - return format_num(self.latest_stats[field], suffix) - - def get_rel(self, field, format=False, suffix=""): - if not field in self.latest_stats: - return "N/A" - - if not format: - if not field in self.reference_stats: - print "REF: " + str(self.reference_stats) - print "BASE: " + str(self.latest_stats) - - return (self.latest_stats[field] - self.reference_stats[field]) - else: - return format_num(self.latest_stats[field] - self.reference_stats[field], suffix) - - # get trend for a field - def get_trend (self, field, use_raw = False, percision = 10.0): - if not field in self.latest_stats: - return 0 - - # not enough history - no trend - if len(self.history) < 5: - return 0 - - # absolute value is too low 0 considered noise - if self.latest_stats[field] < percision: - return 0 - - field_samples = [sample[field] for sample in self.history] - - if use_raw: - return calculate_diff_raw(field_samples) - else: - return calculate_diff(field_samples) - - - def get_trend_gui (self, field, show_value = False, use_raw = False, up_color = 'red', down_color = 'green'): - v = self.get_trend(field, use_raw) - - value = abs(v) - arrow = u'\u25b2' if v > 0 else u'\u25bc' - color = up_color if v > 0 else down_color - - # change in 1% is not meaningful - if value < 1: - return "" - - elif value > 5: - - if show_value: - return format_text(u"{0}{0}{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}{0}{0}".format(arrow), color) - - elif value > 2: - - if show_value: - return format_text(u"{0}{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}{0}".format(arrow), color) - - else: - if show_value: - return format_text(u"{0} {1:.2f}%".format(arrow,v), color) - else: - return format_text(u"{0}".format(arrow), color) - - - -class CGlobalStats(CTRexStats): - - def __init__(self, connection_info, server_version, ports_dict_ref): - super(CGlobalStats, self).__init__() - self.connection_info = connection_info - self.server_version = server_version - self._ports_dict = ports_dict_ref - - def get_stats (self): - stats = {} - - # absolute - stats['cpu_util'] = self.get("m_cpu_util") - stats['tx_bps'] = self.get("m_tx_bps") - stats['tx_pps'] = self.get("m_tx_pps") - - stats['rx_bps'] = self.get("m_rx_bps") - stats['rx_pps'] = self.get("m_rx_pps") - stats['rx_drop_bps'] = self.get("m_rx_drop_bps") - - # relatives - stats['queue_full'] = self.get_rel("m_total_queue_full") - - return stats - - - def generate_stats(self): - return OrderedDict([("connection", "{host}, Port {port}".format(host=self.connection_info.get("server"), - port=self.connection_info.get("sync_port"))), - ("version", "{ver}, UUID: {uuid}".format(ver=self.server_version.get("version", "N/A"), - uuid="N/A")), - - ("cpu_util", u"{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]), - self.get_trend_gui("m_cpu_util", use_raw = True))), - - (" ", ""), - - ("total_tx", u"{0} {1}".format( self.get("m_tx_bps", format=True, suffix="b/sec"), - self.get_trend_gui("m_tx_bps"))), - - ("total_rx", u"{0} {1}".format( self.get("m_rx_bps", format=True, suffix="b/sec"), - self.get_trend_gui("m_rx_bps"))), - - ("total_pps", u"{0} {1}".format( self.get("m_tx_pps", format=True, suffix="pkt/sec"), - self.get_trend_gui("m_tx_pps"))), - - (" ", ""), - - ("drop_rate", "{0}".format( format_num(self.get("m_rx_drop_bps"), - suffix = 'b/sec', - opts = 'green' if (self.get("m_rx_drop_bps")== 0) else 'red'))), - - ("queue_full", "{0}".format( format_num(self.get_rel("m_total_queue_full"), - suffix = 'pkts', - compact = False, - opts = 'green' if (self.get_rel("m_total_queue_full")== 0) else 'red'))), - - ] - ) - -class CPortStats(CTRexStats): - - def __init__(self, port_obj): - super(CPortStats, self).__init__() - self._port_obj = port_obj - - @staticmethod - def __merge_dicts (target, src): - for k, v in src.iteritems(): - if k in target: - target[k] += v - else: - target[k] = v - - - def __add__ (self, x): - if not isinstance(x, CPortStats): - raise TypeError("cannot add non stats object to stats") - - # main stats - if not self.latest_stats: - self.latest_stats = {} - - self.__merge_dicts(self.latest_stats, x.latest_stats) - - # reference stats - if x.reference_stats: - if not self.reference_stats: - self.reference_stats = x.reference_stats.copy() - else: - self.__merge_dicts(self.reference_stats, x.reference_stats) - - # history - if not self.history: - self.history = copy.deepcopy(x.history) - else: - for h1, h2 in zip(self.history, x.history): - self.__merge_dicts(h1, h2) - - return self - - # for port we need to do something smarter - def get_stats (self): - stats = {} - - stats['opackets'] = self.get_rel("opackets") - stats['ipackets'] = self.get_rel("ipackets") - stats['obytes'] = self.get_rel("obytes") - stats['ibytes'] = self.get_rel("ibytes") - stats['oerrors'] = self.get_rel("oerrors") - stats['ierrors'] = self.get_rel("ierrors") - stats['tx_bps'] = self.get("m_total_tx_bps") - stats['tx_pps'] = self.get("m_total_tx_pps") - stats['rx_bps'] = self.get("m_total_rx_bps") - stats['rx_pps'] = self.get("m_total_rx_pps") - - return stats - - - def generate_stats(self): - - state = self._port_obj.get_port_state_name() if self._port_obj else "" - if state == "ACTIVE": - state = format_text(state, 'green', 'bold') - elif state == "PAUSE": - state = format_text(state, 'magenta', 'bold') - else: - state = format_text(state, 'bold') - - return {"owner": self._port_obj.user if self._port_obj else "", - "state": "{0}".format(state), - - "--": " ", - "---": " ", - "----": " ", - "-----": " ", - - "Tx bps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_bps", show_value = False), - self.get("m_total_tx_bps", format = True, suffix = "bps")), - - "Rx bps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_bps", show_value = False), - self.get("m_total_rx_bps", format = True, suffix = "bps")), - - "Tx pps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_pps", show_value = False), - self.get("m_total_tx_pps", format = True, suffix = "pps")), - - "Rx pps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_pps", show_value = False), - self.get("m_total_rx_pps", format = True, suffix = "pps")), - - "opackets" : self.get_rel("opackets"), - "ipackets" : self.get_rel("ipackets"), - "obytes" : self.get_rel("obytes"), - "ibytes" : self.get_rel("ibytes"), - - "tx-bytes": self.get_rel("obytes", format = True, suffix = "B"), - "rx-bytes": self.get_rel("ibytes", format = True, suffix = "B"), - "tx-pkts": self.get_rel("opackets", format = True, suffix = "pkts"), - "rx-pkts": self.get_rel("ipackets", format = True, suffix = "pkts"), - - "oerrors" : format_num(self.get_rel("oerrors"), - compact = False, - opts = 'green' if (self.get_rel("oerrors")== 0) else 'red'), - - "ierrors" : format_num(self.get_rel("ierrors"), - compact = False, - opts = 'green' if (self.get_rel("ierrors")== 0) else 'red'), - - } - - - -if __name__ == "__main__": - pass diff --git a/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py deleted file mode 100644 index 9be20db9..00000000 --- a/scripts/automation/trex_control_plane/common/trex_stl_exceptions.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import sys -from common.text_opts import * - -# basic error for API -class STLError(Exception): - def __init__ (self, msg): - self.msg = str(msg) - - def __str__ (self): - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - - - s = "\n******\n" - s += "Error at {0}:{1}\n\n".format(format_text(fname, 'bold'), format_text(exc_tb.tb_lineno), 'bold') - s += "specific error:\n\n{0}\n".format(format_text(self.msg, 'bold')) - - return s - - def brief (self): - return self.msg - - -# raised when the client state is invalid for operation -class STLStateError(STLError): - def __init__ (self, op, state): - self.msg = "Operation '{0}' is not valid while '{1}'".format(op, state) - - -# port state error -class STLPortStateError(STLError): - def __init__ (self, port, op, state): - self.msg = "Operation '{0}' on port(s) '{1}' is not valid while port(s) '{2}'".format(op, port, state) - - -# raised when argument is not valid for operation -class STLArgumentError(STLError): - def __init__ (self, name, got, valid_values = None, extended = None): - self.msg = "Argument: '{0}' invalid value: '{1}'".format(name, got) - if valid_values: - self.msg += " - valid values are '{0}'".format(valid_values) - - if extended: - self.msg += "\n{0}".format(extended) - -# raised when timeout occurs -class STLTimeoutError(STLError): - def __init__ (self, timeout): - self.msg = "Timeout: operation took more than '{0}' seconds".format(timeout) - - - diff --git a/scripts/automation/trex_control_plane/common/trex_streams.py b/scripts/automation/trex_control_plane/common/trex_streams.py deleted file mode 100755 index c1f1bfa6..00000000 --- a/scripts/automation/trex_control_plane/common/trex_streams.py +++ /dev/null @@ -1,526 +0,0 @@ -#!/router/bin/python - -import external_packages -from client_utils.packet_builder_interface import CTrexPktBuilderInterface -from client_utils.packet_builder import CTRexPktBuilder -from collections import OrderedDict, namedtuple -from client_utils.yaml_utils import * -import trex_stl_exceptions -import dpkt -import struct -import copy -import os -import random -import yaml -import base64 - -StreamPack = namedtuple('StreamPack', ['stream_id', 'stream']) -LoadedStreamList = namedtuple('LoadedStreamList', ['name', 'loaded', 'compiled']) - -class CStreamList(object): - - def __init__(self): - self.streams_list = OrderedDict() - self.yaml_loader = CTRexYAMLLoader(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "rpc_defaults.yaml")) - - def generate_numbered_name (self, name): - prefix = name.rstrip('01234567890') - suffix = name[len(prefix):] - if suffix == "": - n = "_1" - else: - n = int(suffix) + 1 - return prefix + str(n) - - def append_stream(self, name, stream_obj): - assert isinstance(stream_obj, CStream) - - # if name exists simply add numbered suffix to it - while name in self.streams_list: - name = self.generate_numbered_name(name) - - self.streams_list[name]=stream_obj - return name - - def remove_stream(self, name): - popped = self.streams_list.pop(name) - if popped: - for stream_name, stream in self.streams_list.items(): - if stream.next_stream_id == name: - stream.next_stream_id = -1 - try: - rx_stats_stream = getattr(stream.rx_stats, "stream_id") - if rx_stats_stream == name: - # if a referenced stream of rx_stats object deleted, revert to rx stats of current stream - setattr(stream.rx_stats, "stream_id", stream_name) - except AttributeError as e: - continue # - return popped - - def export_to_yaml(self, file_path): - raise NotImplementedError("export_to_yaml method is not implemented, yet") - - def load_yaml(self, file_path, multiplier=1): - # clear all existing streams linked to this object - self.streams_list.clear() - streams_data = load_yaml_to_obj(file_path) - assert isinstance(streams_data, list) - new_streams_data = [] - for stream in streams_data: - stream_name = stream.get("name") - raw_stream = stream.get("stream") - if not stream_name or not raw_stream: - raise ValueError("Provided stream is not according to convention." - "Each stream must be provided as two keys: 'name' and 'stream'. " - "Provided item was:\n {stream}".format(stream)) - new_stream_data = self.yaml_loader.validate_yaml(raw_stream, - "stream", - multiplier= multiplier) - new_streams_data.append(new_stream_data) - new_stream_obj = CStream() - new_stream_obj.load_data(**new_stream_data) - self.append_stream(stream_name, new_stream_obj) - return new_streams_data - - def compile_streams(self): - # first, assign an id to each stream - stream_ids = {} - for idx, stream_name in enumerate(self.streams_list): - stream_ids[stream_name] = idx - - # next, iterate over the streams and transform them from working with names to ids. - # with that build a new dict with old stream_name as the key, and StreamPack as the stored value - compiled_streams = {} - for stream_name, stream in self.streams_list.items(): - tmp_stream = CStreamList._compile_single_stream(stream_name, stream, stream_ids) - compiled_streams[stream_name] = StreamPack(stream_ids.get(stream_name), - tmp_stream) - return compiled_streams - - @staticmethod - def _compile_single_stream(stream_name, stream, id_dict): - # copy the old stream to temporary one, no change to class attributes - tmp_stream = copy.copy(stream) - next_stream_id = id_dict.get(getattr(tmp_stream, "next_stream_id"), -1) - try: - rx_stats_stream_id = id_dict.get(getattr(tmp_stream.rx_stats, "stream_id"), - id_dict.get(stream_name)) - except AttributeError as e: - rx_stats_stream_id = id_dict.get(stream_name) - # assign resolved values to stream object - tmp_stream.next_stream_id = next_stream_id - tmp_stream.rx_stats.stream_id = rx_stats_stream_id - return tmp_stream - - -class CRxStats(object): - - FIELDS = ["seq_enabled", "latency_enabled", "stream_id"] - def __init__(self, enabled=False, **kwargs): - self.enabled = bool(enabled) - for field in CRxStats.FIELDS: - setattr(self, field, kwargs.get(field, False)) - - def dump(self): - if self.enabled: - dump = {"enabled": True} - dump.update({k: getattr(self, k) - for k in CRxStats.FIELDS} - ) - return dump - else: - return {"enabled": False} - - - -class CTxMode(object): - """docstring for CTxMode""" - GENERAL_FIELDS = ["type", "pps"] - FIELDS = {"continuous": [], - "single_burst": ["total_pkts"], - "multi_burst": ["pkts_per_burst", "ibg", "count"]} - - def __init__(self, type, pps=0, **kwargs): - self._MODES = CTxMode.FIELDS.keys() - self.type = type - self.pps = pps - for field in CTxMode.FIELDS.get(self.type): - setattr(self, field, kwargs.get(field, 0)) - - @property - def type(self): - return self._type - - @type.setter - def type(self, type): - if type not in self._MODES: - raise ValueError("Unknown TX mode ('{0}')has been initialized.".format(type)) - self._type = type - self._reset_fields() - - def dump(self): - dump = ({k: getattr(self, k) - for k in CTxMode.GENERAL_FIELDS - }) - dump.update({k: getattr(self, k) - for k in CTxMode.FIELDS.get(self.type) - }) - return dump - - def _reset_fields(self): - for field in CTxMode.FIELDS.get(self.type): - setattr(self, field, 0) - - -class CStream(object): - """docstring for CStream""" - - FIELDS = ["enabled", "self_start", "next_stream_id", "isg", "mode", "rx_stats", "packet", "vm"] - - def __init__(self): - self.is_loaded = False - self._is_compiled = False - self._pkt_bld_obj = CTRexPktBuilder() - for field in CStream.FIELDS: - setattr(self, field, None) - - - def load_data(self, **kwargs): - try: - for k in CStream.FIELDS: - if k == "rx_stats": - rx_stats_data = kwargs[k] - if isinstance(rx_stats_data, dict): - setattr(self, k, CRxStats(**rx_stats_data)) - elif isinstance(rx_stats_data, CRxStats): - setattr(self, k, rx_stats_data) - elif k == "mode": - tx_mode = kwargs[k] - if isinstance(tx_mode, dict): - setattr(self, k, CTxMode(**tx_mode)) - elif isinstance(tx_mode, CTxMode): - setattr(self, k, tx_mode) - elif k == "packet": - if isinstance(kwargs[k], CTRexPktBuilder): - if "vm" not in kwargs: - self.load_packet_obj(kwargs[k]) - break # vm field check is skipped - else: - raise ValueError("When providing packet object with a CTRexPktBuilder, vm parameter " - "should not be supplied") - else: - binary = kwargs[k]["binary"] - if isinstance(binary, str): - - # TODO: load to _pkt_bld_obj also when passed as byte array! - if binary.endswith(".pcap"): - self._pkt_bld_obj.load_packet_from_pcap(binary) - self._pkt_bld_obj.metadata = kwargs[k]["meta"] - self.packet = self._pkt_bld_obj.dump_pkt() - else: - self.packet = {} - self.packet['binary'] = binary - self.packet['meta'] = "" - - else: - raise ValueError("Packet binary attribute has been loaded with unsupported value." - "Supported values are reference to pcap file with SINGLE packet, " - "or a list of unsigned-byte integers") - else: - setattr(self, k, kwargs[k]) - self.is_loaded = True - except KeyError as e: - cause = e.args[0] - raise KeyError("The attribute '{0}' is missing as a field of the CStream object.\n" - "Loaded data must contain all of the following fields: {1}".format(cause, CStream.FIELDS)) - - def load_packet_obj(self, packet_obj): - assert isinstance(packet_obj, CTRexPktBuilder) - self.packet = packet_obj.dump_pkt() - self.vm = packet_obj.get_vm_data() - - def load_packet_from_pcap(self, pcap_path, metadata=''): - with open(pcap_path, 'r') as f: - pcap = dpkt.pcap.Reader(f) - first_packet = True - for _, buf in pcap: - # this is an iterator, can't evaluate the number of files in advance - if first_packet: - self.packet = {"binary": [struct.unpack('B', buf[i:i+1])[0] # represent data as list of 0-255 ints - for i in range(0, len(buf))], - "meta": metadata} # meta data continues without a change. - first_packet = False - else: - raise ValueError("Provided pcap file contains more than single packet.") - # arrive here ONLY if pcap contained SINGLE packet - return - - - def dump(self): - if self.is_loaded: - dump = {} - for key in CStream.FIELDS: - try: - dump[key] = getattr(self, key).dump() # use dump() method of compound object, such TxMode - except AttributeError: - dump[key] = getattr(self, key) - return dump - else: - raise RuntimeError("CStream object isn't loaded with data. Use 'load_data' method.") - - def get_stream_layers(self, depth_limit=Ellipsis): - stream_layers = self._pkt_bld_obj.get_packet_layers(depth_limit) - return "/".join(stream_layers) - - - -# describes a stream DB -class CStreamsDB(object): - - def __init__(self): - self.stream_packs = {} - - def load_yaml_file(self, filename): - - stream_pack_name = filename - if stream_pack_name in self.get_loaded_streams_names(): - self.remove_stream_packs(stream_pack_name) - - stream_list = CStreamList() - loaded_obj = stream_list.load_yaml(filename) - - try: - compiled_streams = stream_list.compile_streams() - rc = self.load_streams(LoadedStreamList(stream_pack_name, - loaded_obj, - [StreamPack(v.stream_id, v.stream.dump()) - for k, v in compiled_streams.items()])) - except Exception as e: - return None - - - return self.get_stream_pack(stream_pack_name) - - def load_streams(self, LoadedStreamList_obj): - if LoadedStreamList_obj.name in self.stream_packs: - return False - else: - self.stream_packs[LoadedStreamList_obj.name] = LoadedStreamList_obj - return True - - def remove_stream_packs(self, *names): - removed_streams = [] - for name in names: - removed = self.stream_packs.pop(name) - if removed: - removed_streams.append(name) - return removed_streams - - def clear(self): - self.stream_packs.clear() - - def get_loaded_streams_names(self): - return self.stream_packs.keys() - - def stream_pack_exists (self, name): - return name in self.get_loaded_streams_names() - - def get_stream_pack(self, name): - if not self.stream_pack_exists(name): - return None - else: - return self.stream_packs.get(name) - - -########################### Simple Streams ########################### -from trex_stl_exceptions import * - -# base class for TX mode -class STLTXMode(object): - def __init__ (self): - self.fields = {} - - def to_json (self): - return self.fields - - -# continuous mode -class STLTXCont(STLTXMode): - - def __init__ (self, pps = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - super(STLTXCont, self).__init__() - - self.fields['type'] = 'continuous' - self.fields['pps'] = pps - - -# single burst mode -class STLTXSingleBurst(STLTXMode): - - def __init__ (self, pps = 1, total_pkts = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - if not isinstance(total_pkts, int): - raise STLArgumentError('total_pkts', total_pkts) - - super(STLTXSingleBurst, self).__init__() - - self.fields['type'] = 'single_burst' - self.fields['pps'] = pps - self.fields['total_pkts'] = total_pkts - - -# multi burst mode -class STLTXMultiBurst(STLTXMode): - - def __init__ (self, - pps = 1, - pkts_per_burst = 1, - ibg = 0.0, - count = 1): - - if not isinstance(pps, (int, float)): - raise STLArgumentError('pps', pps) - - if not isinstance(pkts_per_burst, int): - raise STLArgumentError('pkts_per_burst', pkts_per_burst) - - if not isinstance(ibg, (int, float)): - raise STLArgumentError('ibg', ibg) - - if not isinstance(count, int): - raise STLArgumentError('count', count) - - super(STLTXMultiBurst, self).__init__() - - self.fields['type'] = 'multi_burst' - self.fields['pps'] = pps - self.fields['pkts_per_burst'] = pkts_per_burst - self.fields['ibg'] = ibg - self.fields['count'] = count - - -class STLStream(object): - - def __init__ (self, - packet, - mode = STLTXCont(1), - enabled = True, - self_start = True, - isg = 0.0, - rx_stats = None, - next_stream_id = -1, - stream_id = None): - - # type checking - if not isinstance(mode, STLTXMode): - raise STLArgumentError('mode', mode) - - if not isinstance(packet, CTrexPktBuilderInterface): - raise STLArgumentError('packet', packet) - - if not isinstance(enabled, bool): - raise STLArgumentError('enabled', enabled) - - if not isinstance(self_start, bool): - raise STLArgumentError('self_start', self_start) - - if not isinstance(isg, (int, float)): - raise STLArgumentError('isg', isg) - - if (type(mode) == STLTXCont) and (next_stream_id != -1): - raise STLError("continuous stream cannot have a next stream ID") - - # use a random 31 bit for ID - self.stream_id = stream_id if stream_id is not None else random.getrandbits(31) - - self.fields = {} - - # basic fields - self.fields['enabled'] = enabled - self.fields['self_start'] = self_start - self.fields['isg'] = isg - - self.fields['next_stream_id'] = next_stream_id - - # mode - self.fields['mode'] = mode.to_json() - - packet.compile() - - # packet and VM - self.fields['packet'] = packet.dump_pkt() - self.fields['vm'] = packet.get_vm_data() - - self.fields['rx_stats'] = {} - if not rx_stats: - self.fields['rx_stats']['enabled'] = False - - - def __str__ (self): - return json.dumps(self.fields, indent = 4, separators=(',', ': '), sort_keys = True) - - def to_json (self): - return self.fields - - def get_id (self): - return self.stream_id - - @staticmethod - def dump_to_yaml (yaml_file, stream_list): - - # type check - if isinstance(stream_list, STLStream): - stream_list = [stream_list] - - if not all([isinstance(stream, STLStream) for stream in stream_list]): - raise STLArgumentError('stream_list', stream_list) - - - names = {} - for i, stream in enumerate(stream_list): - names[stream.get_id()] = "stream-{0}".format(i) - - yaml_lst = [] - for stream in stream_list: - - fields = dict(stream.fields) - - # handle the next stream id - if fields['next_stream_id'] == -1: - del fields['next_stream_id'] - - else: - if not stream.get_id() in names: - raise STLError('broken dependencies in stream list') - - fields['next_stream'] = names[stream.get_id()] - - # add to list - yaml_lst.append({'name': names[stream.get_id()], 'stream': fields}) - - # write to file - x = yaml.dump(yaml_lst, default_flow_style=False) - with open(yaml_file, 'w') as f: - f.write(x) - return x - - -# REMOVE ME when can - convert from stream pack to a simple stream -class HACKSTLStream(STLStream): - def __init__ (self, stream_pack): - if not isinstance(stream_pack, StreamPack): - raise Exception("internal error") - - packet = CTRexPktBuilder() - packet.load_from_stream_obj(stream_pack.stream) - super(HACKSTLStream, self).__init__(packet, stream_id = stream_pack.stream_id) - - self.fields = stream_pack.stream diff --git a/scripts/automation/trex_control_plane/common/trex_types.py b/scripts/automation/trex_control_plane/common/trex_types.py deleted file mode 100644 index a7ddacea..00000000 --- a/scripts/automation/trex_control_plane/common/trex_types.py +++ /dev/null @@ -1,95 +0,0 @@ - -from collections import namedtuple -from common.text_opts import * - -RpcCmdData = namedtuple('RpcCmdData', ['method', 'params']) - -class RpcResponseStatus(namedtuple('RpcResponseStatus', ['success', 'id', 'msg'])): - __slots__ = () - def __str__(self): - return "{id:^3} - {msg} ({stat})".format(id=self.id, - msg=self.msg, - stat="success" if self.success else "fail") - -# simple class to represent complex return value -class RC(): - - def __init__ (self, rc = None, data = None, is_warn = False): - self.rc_list = [] - - if (rc != None): - tuple_rc = namedtuple('RC', ['rc', 'data', 'is_warn']) - self.rc_list.append(tuple_rc(rc, data, is_warn)) - - def __nonzero__ (self): - return self.good() - - - def add (self, rc): - self.rc_list += rc.rc_list - - def good (self): - return all([x.rc for x in self.rc_list]) - - def bad (self): - return not self.good() - - def warn (self): - return any([x.is_warn for x in self.rc_list]) - - def data (self): - d = [x.data if x.rc else "" for x in self.rc_list] - return (d if len(d) != 1 else d[0]) - - def err (self): - e = [x.data if not x.rc else "" for x in self.rc_list] - return (e if len(e) != 1 else e[0]) - - def __str__ (self): - s = "" - for x in self.rc_list: - if x.data: - s += format_text("\n{0}".format(x.data), 'bold') - return s - - def prn_func (self, msg, newline = True): - if newline: - print msg - else: - print msg, - - def annotate (self, log_func = None, desc = None, show_status = True): - - if not log_func: - log_func = self.prn_func - - if desc: - log_func(format_text('\n{:<60}'.format(desc), 'bold'), newline = False) - else: - log_func("") - - if self.bad(): - # print all the errors - print "" - for x in self.rc_list: - if not x.rc: - log_func(format_text("\n{0}".format(x.data), 'bold')) - - print "" - if show_status: - log_func(format_text("[FAILED]\n", 'red', 'bold')) - - - else: - if show_status: - log_func(format_text("[SUCCESS]\n", 'green', 'bold')) - - -def RC_OK(data = ""): - return RC(True, data) - -def RC_ERR (err): - return RC(False, err) - -def RC_WARN (warn): - return RC(True, warn, is_warn = True) diff --git a/scripts/automation/trex_control_plane/console/__init__.py b/scripts/automation/trex_control_plane/console/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/automation/trex_control_plane/console/trex_console.py b/scripts/automation/trex_control_plane/console/trex_console.py index c8624626..45428b89 100755 --- a/scripts/automation/trex_control_plane/console/trex_console.py +++ b/scripts/automation/trex_control_plane/console/trex_console.py @@ -27,14 +27,16 @@ import string import os import sys import tty, termios -import trex_root_path -from common.trex_streams import * -from client.trex_stateless_client import STLClient, LoggerApi -from common.text_opts import * -from client_utils.general_utils import user_input, get_current_user -from client_utils import parsing_opts + +from trex_control_plane.stl.api import * + +from trex_control_plane.common.text_opts import * +from trex_control_plane.client_utils.general_utils import user_input, get_current_user +from trex_control_plane.client_utils import parsing_opts + + import trex_tui -from common.trex_stl_exceptions import * + from functools import wraps __version__ = "1.1" diff --git a/scripts/automation/trex_control_plane/console/trex_tui.py b/scripts/automation/trex_control_plane/console/trex_tui.py index 1e22b005..1ecf0868 100644 --- a/scripts/automation/trex_control_plane/console/trex_tui.py +++ b/scripts/automation/trex_control_plane/console/trex_tui.py @@ -2,13 +2,15 @@ import termios import sys import os import time -from common.text_opts import * -from common import trex_stats -from client_utils import text_tables from collections import OrderedDict import datetime from cStringIO import StringIO -from client.trex_stateless_client import STLError + +from common.text_opts import * +from client_utils import text_tables + +# for STL exceptions +from trex_control_plane.stl.api import * class SimpleBar(object): def __init__ (self, desc, pattern): @@ -61,7 +63,7 @@ class TrexTUIDashBoard(TrexTUIPanel): def show (self): - stats = self.stateless_client._get_formatted_stats(self.ports, trex_stats.COMPACT) + stats = self.stateless_client._get_formatted_stats(self.ports) # print stats to screen for stat_type, stat_data in stats.iteritems(): text_tables.print_table_with_header(stat_data.text_table, stat_type) @@ -148,7 +150,7 @@ class TrexTUIPort(TrexTUIPanel): def show (self): - stats = self.stateless_client._get_formatted_stats([self.port_id], trex_stats.COMPACT) + stats = self.stateless_client._get_formatted_stats([self.port_id]) # print stats to screen for stat_type, stat_data in stats.iteritems(): text_tables.print_table_with_header(stat_data.text_table, stat_type) diff --git a/scripts/automation/trex_control_plane/stl/__init__.py b/scripts/automation/trex_control_plane/stl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/automation/trex_control_plane/stl/api.py b/scripts/automation/trex_control_plane/stl/api.py new file mode 100644 index 00000000..c12628b5 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/api.py @@ -0,0 +1,31 @@ + +# get external libs +import trex_control_plane.client_utils.external_packages + +# client and exceptions +from trex_stl_exceptions import * +from trex_stl_client import STLClient, LoggerApi + +# streams +from trex_stl_streams import * + +# packet builder +from trex_stl_packet_builder_scapy import * +from scapy.all import * + +# packet builder +STLPktBuilder = CScapyTRexPktBuilder + +# VM +STLVmFlowVar = CTRexVmDescFlowVar +STLVmWriteFlowVar = CTRexVmDescWrFlowVar +STLVmFixIpv4 = CTRexVmDescFixIpv4 +STLVmTrimPktSize = CTRexVmDescTrimPktSize +STLVmTupleGen = CTRexVmDescTupleGen + + +# simulator +from trex_stl_sim import STLSim + +# std lib (various lib functions) +from trex_stl_std import * diff --git a/scripts/automation/trex_control_plane/stl/rpc_defaults.yaml b/scripts/automation/trex_control_plane/stl/rpc_defaults.yaml new file mode 100644 index 00000000..ad814b3e --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/rpc_defaults.yaml @@ -0,0 +1,124 @@ +############################################################## +#### TRex RPC stream list default values #### +############################################################## + +# this document is based on TRex RPC server spec and its fields: +# http://trex-tgn.cisco.com/trex/doc/trex_rpc_server_spec.html + +### HOW TO READ THIS FILE +# 1. Each key represents an object type +# 2. Each value can be either a value field or another object +# 2.1. If a value field, read as: +# + type: type of field +# + has_default: if the value has any default +# + default: the default value (Only appears if has_default field is 'YES') +# 2.2. If an object type, jump to corresponding object key. +# 3. If an object has more than one instance type, another layer with the type shall be added. +# For example, 'mode' object has 3 types: 'continuous', 'single_burst', 'multi_burst' +# So, 3 mode objects will be defined, named: +# - mode['continuous'] +# - mode['single_burst'] +# - mode['multi_burst'] +# In this case, there's no default for the 'type' field on the object +# 4. Some values has 'multiply' property attached. +# In such case, the loaded value will be multiplied by the multiplier +# For example, if the mode's 'pps' field value is 10, and its multiplier is 5, +# the loaded pps value will be 10*5=50 +# 5. Any object type must be listed by the user, even if all its field are defaults. +# The most basic option would be to declare the object with "[]", which stands for empty object in YAML syntax. + + +stream: + enabled: + type: boolean + has_default: YES + default: True + self_start: + type: boolean + has_default: YES + default: True + isg: + type: [int, double, string] + has_default: YES + default: 0.0 + next_stream_id: + type: string # string to allow naming binding + has_default: YES + default: -1 # no next streams + packet: + type: object + mode: + type: object + vm: + type: object + rx_stats: + type: object + +packet: + binary: + type: [array,string] + has_default: NO + meta: + type: string + has_default: YES + default: "" + +mode: + continuous: + pps: + type: [int, double] + has_default: NO + multiply: YES + single_burst: + pps: + type: [int, double] + has_default: NO + multiply: YES + total_pkts: + type: int + has_default: NO + multi_burst: + pps: + type: [int, double] + has_default: NO + multiply: YES + pkts_per_burst: + type: int + has_default: NO + ibg: + type: [int, double, string] + has_default: YES + default: 100.0 + count: + type: int + has_default: YES + default: 0 # loop forever + +rx_stats: + enabled: + type: boolean + has_default: YES + default: False + stream_id: + type: string + has_default: YES + default: False # use related stream_id + seq_enabled: + type: boolean + has_default: YES + default: False + latency_enabled: + type: boolean + has_default: YES + default: False + +vm: + instructions: + type: array + has_default: YES + default: [] + split_by_var: + type: string + has_default: YES + default: "" + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_async_client.py new file mode 100644 index 00000000..9b3b9577 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_async_client.py @@ -0,0 +1,322 @@ +#!/router/bin/python + +import json +import threading +import time +import datetime +import zmq +import re +import random + +from trex_stl_jsonrpc_client import JsonRpcClient, BatchMessage + +from common.text_opts import * +from trex_stl_stats import * +from trex_stl_types import * + +# basic async stats class +class CTRexAsyncStats(object): + def __init__ (self): + self.ref_point = None + self.current = {} + self.last_update_ts = datetime.datetime.now() + + def update (self, snapshot): + + #update + self.last_update_ts = datetime.datetime.now() + + self.current = snapshot + + if self.ref_point == None: + self.ref_point = self.current + + def clear(self): + self.ref_point = self.current + + + def get(self, field, format=False, suffix=""): + + if not field in self.current: + return "N/A" + + if not format: + return self.current[field] + else: + return format_num(self.current[field], suffix) + + def get_rel (self, field, format=False, suffix=""): + if not field in self.current: + return "N/A" + + if not format: + return (self.current[field] - self.ref_point[field]) + else: + return format_num(self.current[field] - self.ref_point[field], suffix) + + + # return true if new data has arrived in the past 2 seconds + def is_online (self): + delta_ms = (datetime.datetime.now() - self.last_update_ts).total_seconds() * 1000 + return (delta_ms < 2000) + +# describes the general stats provided by TRex +class CTRexAsyncStatsGeneral(CTRexAsyncStats): + def __init__ (self): + super(CTRexAsyncStatsGeneral, self).__init__() + + +# per port stats +class CTRexAsyncStatsPort(CTRexAsyncStats): + def __init__ (self): + super(CTRexAsyncStatsPort, self).__init__() + + def get_stream_stats (self, stream_id): + return None + +# stats manager +class CTRexAsyncStatsManager(): + def __init__ (self): + + self.general_stats = CTRexAsyncStatsGeneral() + self.port_stats = {} + + + def get_general_stats(self): + return self.general_stats + + def get_port_stats (self, port_id): + + if not str(port_id) in self.port_stats: + return None + + return self.port_stats[str(port_id)] + + + def update(self, data): + self.__handle_snapshot(data) + + def __handle_snapshot(self, snapshot): + + general_stats = {} + port_stats = {} + + # filter the values per port and general + for key, value in snapshot.iteritems(): + + # match a pattern of ports + m = re.search('(.*)\-([0-8])', key) + if m: + + port_id = m.group(2) + field_name = m.group(1) + + if not port_id in port_stats: + port_stats[port_id] = {} + + port_stats[port_id][field_name] = value + + else: + # no port match - general stats + general_stats[key] = value + + # update the general object with the snapshot + self.general_stats.update(general_stats) + + # update all ports + for port_id, data in port_stats.iteritems(): + + if not port_id in self.port_stats: + self.port_stats[port_id] = CTRexAsyncStatsPort() + + self.port_stats[port_id].update(data) + + + + + +class CTRexAsyncClient(): + def __init__ (self, server, port, stateless_client): + + self.port = port + self.server = server + + self.stateless_client = stateless_client + + self.event_handler = stateless_client.event_handler + self.logger = self.stateless_client.logger + + self.raw_snapshot = {} + + self.stats = CTRexAsyncStatsManager() + + self.last_data_recv_ts = 0 + self.async_barrier = None + + self.connected = False + + # connects the async channel + def connect (self): + + if self.connected: + self.disconnect() + + self.tr = "tcp://{0}:{1}".format(self.server, self.port) + + # Socket to talk to server + self.context = zmq.Context() + self.socket = self.context.socket(zmq.SUB) + + + # before running the thread - mark as active + self.active = True + self.t = threading.Thread(target = self._run) + + # kill this thread on exit and don't add it to the join list + self.t.setDaemon(True) + self.t.start() + + self.connected = True + + rc = self.barrier() + if not rc: + self.disconnect() + return rc + + return RC_OK() + + + + + # disconnect + def disconnect (self): + if not self.connected: + return + + # signal that the context was destroyed (exit the thread loop) + self.context.term() + + # mark for join and join + self.active = False + self.t.join() + + # done + self.connected = False + + + # thread function + def _run (self): + + # socket must be created on the same thread + self.socket.setsockopt(zmq.SUBSCRIBE, '') + self.socket.setsockopt(zmq.RCVTIMEO, 5000) + self.socket.connect(self.tr) + + got_data = False + + while self.active: + try: + + line = self.socket.recv_string() + self.last_data_recv_ts = time.time() + + # signal once + if not got_data: + self.event_handler.on_async_alive() + got_data = True + + + # got a timeout - mark as not alive and retry + except zmq.Again: + + # signal once + if got_data: + self.event_handler.on_async_dead() + got_data = False + + continue + + except zmq.ContextTerminated: + # outside thread signaled us to exit + break + + msg = json.loads(line) + + name = msg['name'] + data = msg['data'] + type = msg['type'] + self.raw_snapshot[name] = data + + self.__dispatch(name, type, data) + + + # closing of socket must be from the same thread + self.socket.close(linger = 0) + + + # did we get info for the last 3 seconds ? + def is_alive (self): + if self.last_data_recv_ts == None: + return False + + return ( (time.time() - self.last_data_recv_ts) < 3 ) + + def get_stats (self): + return self.stats + + def get_raw_snapshot (self): + return self.raw_snapshot + + # dispatch the message to the right place + def __dispatch (self, name, type, data): + # stats + if name == "trex-global": + self.event_handler.handle_async_stats_update(data) + + # events + elif name == "trex-event": + self.event_handler.handle_async_event(type, data) + + # barriers + elif name == "trex-barrier": + self.handle_async_barrier(type, data) + else: + pass + + + # async barrier handling routine + def handle_async_barrier (self, type, data): + if self.async_barrier['key'] == type: + self.async_barrier['ack'] = True + + + # block on barrier for async channel + def barrier(self, timeout = 5): + + # set a random key + key = random.getrandbits(32) + self.async_barrier = {'key': key, 'ack': False} + + # expr time + expr = time.time() + timeout + + while not self.async_barrier['ack']: + + # inject + rc = self.stateless_client._transmit("publish_now", params = {'key' : key}) + if not rc: + return rc + + # fast loop + for i in xrange(0, 100): + if self.async_barrier['ack']: + break + time.sleep(0.001) + + if time.time() > expr: + return RC_ERR("*** [subscriber] - timeout - no data flow from server at : " + self.tr) + + return RC_OK() + + + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_client.py new file mode 100644 index 00000000..08f640b5 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_client.py @@ -0,0 +1,2020 @@ +#!/router/bin/python + +# for API usage the path name must be full +from trex_control_plane.stl.trex_stl_exceptions import * +#from trex_control_plane.stl.trex_stl_streams import * +from trex_stl_streams import * + +from trex_stl_jsonrpc_client import JsonRpcClient, BatchMessage +import trex_stl_stats + +from trex_stl_port import Port +from trex_stl_types import * +from trex_stl_async_client import CTRexAsyncClient + +from trex_control_plane.client_utils import parsing_opts, text_tables, general_utils +from trex_control_plane.common.text_opts import * + + +from collections import namedtuple +from yaml import YAMLError +import time +import datetime +import re +import random +import json + +############################ logger ############################# +############################ ############################# +############################ ############################# + +# logger API for the client +class LoggerApi(object): + # verbose levels + VERBOSE_QUIET = 0 + VERBOSE_REGULAR = 1 + VERBOSE_HIGH = 2 + + def __init__(self): + self.level = LoggerApi.VERBOSE_REGULAR + + # implemented by specific logger + def write(self, msg, newline = True): + raise Exception("implement this") + + # implemented by specific logger + def flush(self): + raise Exception("implement this") + + def set_verbose (self, level): + if not level in xrange(self.VERBOSE_QUIET, self.VERBOSE_HIGH + 1): + raise ValueError("bad value provided for logger") + + self.level = level + + def get_verbose (self): + return self.level + + + def check_verbose (self, level): + return (self.level >= level) + + + # simple log message with verbose + def log (self, msg, level = VERBOSE_REGULAR, newline = True): + if not self.check_verbose(level): + return + + self.write(msg, newline) + + # logging that comes from async event + def async_log (self, msg, level = VERBOSE_REGULAR, newline = True): + self.log(msg, level, newline) + + + def pre_cmd (self, desc): + self.log(format_text('\n{:<60}'.format(desc), 'bold'), newline = False) + self.flush() + + def post_cmd (self, rc): + if rc: + self.log(format_text("[SUCCESS]\n", 'green', 'bold')) + else: + self.log(format_text("[FAILED]\n", 'red', 'bold')) + + + def log_cmd (self, desc): + self.pre_cmd(desc) + self.post_cmd(True) + + + # supress object getter + def supress (self): + class Supress(object): + def __init__ (self, logger): + self.logger = logger + + def __enter__ (self): + self.saved_level = self.logger.get_verbose() + self.logger.set_verbose(LoggerApi.VERBOSE_QUIET) + + def __exit__ (self, type, value, traceback): + self.logger.set_verbose(self.saved_level) + + return Supress(self) + + + +# default logger - to stdout +class DefaultLogger(LoggerApi): + + def __init__ (self): + super(DefaultLogger, self).__init__() + + def write (self, msg, newline = True): + if newline: + print msg + else: + print msg, + + def flush (self): + sys.stdout.flush() + + +############################ async event hander ############################# +############################ ############################# +############################ ############################# + +# handles different async events given to the client +class AsyncEventHandler(object): + + def __init__ (self, client): + self.client = client + self.logger = self.client.logger + + self.events = [] + + # public functions + + def get_events (self): + return self.events + + + def clear_events (self): + self.events = [] + + + def on_async_dead (self): + if self.client.connected: + msg = 'lost connection to server' + self.__add_event_log(msg, 'local', True) + self.client.connected = False + + + def on_async_alive (self): + pass + + + # handles an async stats update from the subscriber + def handle_async_stats_update(self, dump_data): + global_stats = {} + port_stats = {} + + # filter the values per port and general + for key, value in dump_data.iteritems(): + # match a pattern of ports + m = re.search('(.*)\-([0-8])', key) + if m: + port_id = int(m.group(2)) + field_name = m.group(1) + if self.client.ports.has_key(port_id): + if not port_id in port_stats: + port_stats[port_id] = {} + port_stats[port_id][field_name] = value + else: + continue + else: + # no port match - general stats + global_stats[key] = value + + # update the general object with the snapshot + self.client.global_stats.update(global_stats) + + # update all ports + for port_id, data in port_stats.iteritems(): + self.client.ports[port_id].port_stats.update(data) + + + # dispatcher for server async events (port started, port stopped and etc.) + def handle_async_event (self, type, data): + # DP stopped + show_event = False + + # port started + if (type == 0): + port_id = int(data['port_id']) + ev = "Port {0} has started".format(port_id) + self.__async_event_port_started(port_id) + + # port stopped + elif (type == 1): + port_id = int(data['port_id']) + ev = "Port {0} has stopped".format(port_id) + + # call the handler + self.__async_event_port_stopped(port_id) + + + # port paused + elif (type == 2): + port_id = int(data['port_id']) + ev = "Port {0} has paused".format(port_id) + + # call the handler + self.__async_event_port_paused(port_id) + + # port resumed + elif (type == 3): + port_id = int(data['port_id']) + ev = "Port {0} has resumed".format(port_id) + + # call the handler + self.__async_event_port_resumed(port_id) + + # port finished traffic + elif (type == 4): + port_id = int(data['port_id']) + ev = "Port {0} job done".format(port_id) + + # call the handler + self.__async_event_port_stopped(port_id) + show_event = True + + # port was stolen... + elif (type == 5): + session_id = data['session_id'] + + # false alarm, its us + if session_id == self.client.session_id: + return + + port_id = int(data['port_id']) + who = data['who'] + + ev = "Port {0} was forcely taken by '{1}'".format(port_id, who) + + # call the handler + self.__async_event_port_forced_acquired(port_id) + show_event = True + + # server stopped + elif (type == 100): + ev = "Server has stopped" + self.__async_event_server_stopped() + show_event = True + + + else: + # unknown event - ignore + return + + + self.__add_event_log(ev, 'server', show_event) + + + # private functions + + def __async_event_port_stopped (self, port_id): + self.client.ports[port_id].async_event_port_stopped() + + + def __async_event_port_started (self, port_id): + self.client.ports[port_id].async_event_port_started() + + + def __async_event_port_paused (self, port_id): + self.client.ports[port_id].async_event_port_paused() + + + def __async_event_port_resumed (self, port_id): + self.client.ports[port_id].async_event_port_resumed() + + + def __async_event_port_forced_acquired (self, port_id): + self.client.ports[port_id].async_event_forced_acquired() + + + def __async_event_server_stopped (self): + self.client.connected = False + + + # add event to log + def __add_event_log (self, msg, ev_type, show = False): + + if ev_type == "server": + prefix = "[server]" + elif ev_type == "local": + prefix = "[local]" + + ts = time.time() + st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') + self.events.append("{:<10} - {:^8} - {:}".format(st, prefix, format_text(msg, 'bold'))) + + if show: + self.logger.async_log(format_text("\n\n{:^8} - {:}".format(prefix, format_text(msg, 'bold')))) + + + + + +############################ RPC layer ############################# +############################ ############################# +############################ ############################# + +class CCommLink(object): + """describes the connectivity of the stateless client method""" + def __init__(self, server="localhost", port=5050, virtual=False, prn_func = None): + self.virtual = virtual + self.server = server + self.port = port + self.rpc_link = JsonRpcClient(self.server, self.port, prn_func) + + @property + def is_connected(self): + if not self.virtual: + return self.rpc_link.connected + else: + return True + + def get_server (self): + return self.server + + def get_port (self): + return self.port + + def connect(self): + if not self.virtual: + return self.rpc_link.connect() + + def disconnect(self): + if not self.virtual: + return self.rpc_link.disconnect() + + def transmit(self, method_name, params={}): + if self.virtual: + self._prompt_virtual_tx_msg() + _, msg = self.rpc_link.create_jsonrpc_v2(method_name, params) + print msg + return + else: + return self.rpc_link.invoke_rpc_method(method_name, params) + + def transmit_batch(self, batch_list): + if self.virtual: + self._prompt_virtual_tx_msg() + print [msg + for _, msg in [self.rpc_link.create_jsonrpc_v2(command.method, command.params) + for command in batch_list]] + else: + batch = self.rpc_link.create_batch() + for command in batch_list: + batch.add(command.method, command.params) + # invoke the batch + return batch.invoke() + + def _prompt_virtual_tx_msg(self): + print "Transmitting virtually over tcp://{server}:{port}".format(server=self.server, + port=self.port) + + + +############################ client ############################# +############################ ############################# +############################ ############################# + +class STLClient(object): + """docstring for STLClient""" + + def __init__(self, + username = general_utils.get_current_user(), + server = "localhost", + sync_port = 4501, + async_port = 4500, + verbose_level = LoggerApi.VERBOSE_QUIET, + logger = None, + virtual = False): + + + self.username = username + + # init objects + self.ports = {} + self.server_version = {} + self.system_info = {} + self.session_id = random.getrandbits(32) + self.connected = False + + # logger + self.logger = DefaultLogger() if not logger else logger + + # initial verbose + self.logger.set_verbose(verbose_level) + + # low level RPC layer + self.comm_link = CCommLink(server, + sync_port, + virtual, + self.logger) + + # async event handler manager + self.event_handler = AsyncEventHandler(self) + + # async subscriber level + self.async_client = CTRexAsyncClient(server, + async_port, + self) + + + + + # stats + self.connection_info = {"username": username, + "server": server, + "sync_port": sync_port, + "async_port": async_port, + "virtual": virtual} + + + self.global_stats = trex_stl_stats.CGlobalStats(self.connection_info, + self.server_version, + self.ports) + + self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats, + self.ports) + + + + ############# private functions - used by the class itself ########### + + # some preprocessing for port argument + def __ports (self, port_id_list): + + # none means all + if port_id_list == None: + return range(0, self.get_port_count()) + + # always list + if isinstance(port_id_list, int): + port_id_list = [port_id_list] + + if not isinstance(port_id_list, list): + raise ValueError("bad port id list: {0}".format(port_id_list)) + + for port_id in port_id_list: + if not isinstance(port_id, int) or (port_id < 0) or (port_id > self.get_port_count()): + raise ValueError("bad port id {0}".format(port_id)) + + return port_id_list + + + # sync ports + def __sync_ports (self, port_id_list = None, force = False): + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].sync()) + + return rc + + # acquire ports, if port_list is none - get all + def __acquire (self, port_id_list = None, force = False): + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].acquire(force)) + + return rc + + # release ports + def __release (self, port_id_list = None): + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].release()) + + return rc + + + def __add_streams(self, stream_list, port_id_list = None): + + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].add_streams(stream_list)) + + return rc + + + + def __remove_streams(self, stream_id_list, port_id_list = None): + + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].remove_streams(stream_id_list)) + + return rc + + + + def __remove_all_streams(self, port_id_list = None): + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].remove_all_streams()) + + return rc + + + def __get_stream(self, stream_id, port_id, get_pkt = False): + + return self.ports[port_id].get_stream(stream_id) + + + def __get_all_streams(self, port_id, get_pkt = False): + + return self.ports[port_id].get_all_streams() + + + def __get_stream_id_list(self, port_id): + + return self.ports[port_id].get_stream_id_list() + + + def __start (self, multiplier, duration, port_id_list = None, force = False): + + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].start(multiplier, duration, force)) + + return rc + + + def __resume (self, port_id_list = None, force = False): + + port_id_list = self.__ports(port_id_list) + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].resume()) + + return rc + + def __pause (self, port_id_list = None, force = False): + + port_id_list = self.__ports(port_id_list) + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].pause()) + + return rc + + + def __stop (self, port_id_list = None, force = False): + + port_id_list = self.__ports(port_id_list) + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].stop(force)) + + return rc + + + def __update (self, mult, port_id_list = None, force = False): + + port_id_list = self.__ports(port_id_list) + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].update(mult, force)) + + return rc + + + def __validate (self, port_id_list = None): + port_id_list = self.__ports(port_id_list) + + rc = RC() + + for port_id in port_id_list: + rc.add(self.ports[port_id].validate()) + + return rc + + + + # connect to server + def __connect(self): + + # first disconnect if already connected + if self.is_connected(): + self.__disconnect() + + # clear this flag + self.connected = False + + # connect sync channel + self.logger.pre_cmd("Connecting to RPC server on {0}:{1}".format(self.connection_info['server'], self.connection_info['sync_port'])) + rc = self.comm_link.connect() + self.logger.post_cmd(rc) + + if not rc: + return rc + + # version + rc = self._transmit("get_version") + if not rc: + return rc + + + self.server_version = rc.data() + self.global_stats.server_version = rc.data() + + # cache system info + rc = self._transmit("get_system_info") + if not rc: + return rc + + self.system_info = rc.data() + + # cache supported commands + rc = self._transmit("get_supported_cmds") + if not rc: + return rc + + self.supported_cmds = rc.data() + + # create ports + for port_id in xrange(self.system_info["port_count"]): + speed = self.system_info['ports'][port_id]['speed'] + driver = self.system_info['ports'][port_id]['driver'] + + self.ports[port_id] = Port(port_id, + speed, + driver, + self.username, + self.comm_link, + self.session_id) + + + # sync the ports + rc = self.__sync_ports() + if not rc: + return rc + + + # connect async channel + self.logger.pre_cmd("connecting to publisher server on {0}:{1}".format(self.connection_info['server'], self.connection_info['async_port'])) + rc = self.async_client.connect() + self.logger.post_cmd(rc) + + if not rc: + return rc + + self.connected = True + + return RC_OK() + + + # disconenct from server + def __disconnect(self, release_ports = True): + # release any previous acquired ports + if self.is_connected() and release_ports: + self.__release(self.get_acquired_ports()) + + self.comm_link.disconnect() + self.async_client.disconnect() + + self.connected = False + + return RC_OK() + + + # clear stats + def __clear_stats(self, port_id_list, clear_global): + + for port_id in port_id_list: + self.ports[port_id].clear_stats() + + if clear_global: + self.global_stats.clear_stats() + + self.logger.log_cmd("clearing stats on port(s) {0}:".format(port_id_list)) + + return RC + + + # get stats + def __get_stats (self, port_id_list): + stats = {} + + stats['global'] = self.global_stats.get_stats() + + total = {} + for port_id in port_id_list: + port_stats = self.ports[port_id].get_stats() + stats[port_id] = port_stats + + for k, v in port_stats.iteritems(): + if not k in total: + total[k] = v + else: + total[k] += v + + stats['total'] = total + + return stats + + + ############ functions used by other classes but not users ############## + + def _verify_port_id_list (self, port_id_list): + # check arguments + if not isinstance(port_id_list, list): + return RC_ERR("ports should be an instance of 'list' not {0}".format(type(port_id_list))) + + # all ports are valid ports + if not port_id_list or not all([port_id in self.get_all_ports() for port_id in port_id_list]): + return RC_ERR("") + + return RC_OK() + + def _validate_port_list(self, port_id_list): + if not isinstance(port_id_list, list): + return False + + # check each item of the sequence + return (port_id_list and all([port_id in self.get_all_ports() for port_id in port_id_list])) + + + + # transmit request on the RPC link + def _transmit(self, method_name, params={}): + return self.comm_link.transmit(method_name, params) + + # transmit batch request on the RPC link + def _transmit_batch(self, batch_list): + return self.comm_link.transmit_batch(batch_list) + + # stats + def _get_formatted_stats(self, port_id_list, stats_mask = trex_stl_stats.COMPACT): + stats_opts = trex_stl_stats.ALL_STATS_OPTS.intersection(stats_mask) + + stats_obj = {} + for stats_type in stats_opts: + stats_obj.update(self.stats_generator.generate_single_statistic(port_id_list, stats_type)) + + return stats_obj + + def _get_streams(self, port_id_list, streams_mask=set()): + + streams_obj = self.stats_generator.generate_streams_info(port_id_list, streams_mask) + + return streams_obj + + + def _invalidate_stats (self, port_id_list): + for port_id in port_id_list: + self.ports[port_id].invalidate_stats() + + self.global_stats.invalidate() + + return RC_OK() + + + + + + ################################# + # ------ private methods ------ # + @staticmethod + def __get_mask_keys(ok_values={True}, **kwargs): + masked_keys = set() + for key, val in kwargs.iteritems(): + if val in ok_values: + masked_keys.add(key) + return masked_keys + + @staticmethod + def __filter_namespace_args(namespace, ok_values): + return {k: v for k, v in namespace.__dict__.items() if k in ok_values} + + + # API decorator - double wrap because of argument + def __api_check(connected = True): + + def wrap (f): + def wrap2(*args, **kwargs): + client = args[0] + + func_name = f.__name__ + + # check connection + if connected and not client.is_connected(): + raise STLStateError(func_name, 'disconnected') + + ret = f(*args, **kwargs) + return ret + return wrap2 + + return wrap + + + + ############################ API ############################# + ############################ ############################# + ############################ ############################# + def __enter__ (self): + self.connect() + self.acquire(force = True) + self.reset() + return self + + def __exit__ (self, type, value, traceback): + if self.get_active_ports(): + self.stop(self.get_active_ports()) + self.disconnect() + + ############################ Getters ############################# + ############################ ############################# + ############################ ############################# + + + # return verbose level of the logger + def get_verbose (self): + return self.logger.get_verbose() + + # is the client on read only mode ? + def is_all_ports_acquired (self): + return not (self.get_all_ports() == self.get_acquired_ports()) + + # is the client connected ? + def is_connected (self): + return self.connected and self.comm_link.is_connected + + + # get connection info + def get_connection_info (self): + return self.connection_info + + + # get supported commands by the server + def get_server_supported_cmds(self): + return self.supported_cmds + + # get server version + def get_server_version(self): + return self.server_version + + # get server system info + def get_server_system_info(self): + return self.system_info + + # get port count + def get_port_count(self): + return len(self.ports) + + + # returns the port object + def get_port (self, port_id): + port = self.ports.get(port_id, None) + if (port != None): + return port + else: + raise STLArgumentError('port id', port_id, valid_values = self.get_all_ports()) + + + # get all ports as IDs + def get_all_ports (self): + return self.ports.keys() + + # get all acquired ports + def get_acquired_ports(self): + return [port_id + for port_id, port_obj in self.ports.iteritems() + if port_obj.is_acquired()] + + # get all active ports (TX or pause) + def get_active_ports(self): + return [port_id + for port_id, port_obj in self.ports.iteritems() + if port_obj.is_active()] + + # get paused ports + def get_paused_ports (self): + return [port_id + for port_id, port_obj in self.ports.iteritems() + if port_obj.is_paused()] + + # get all TX ports + def get_transmitting_ports (self): + return [port_id + for port_id, port_obj in self.ports.iteritems() + if port_obj.is_transmitting()] + + + # get stats + def get_stats (self, ports = None, async_barrier = True): + # by default use all ports + if ports == None: + ports = self.get_acquired_ports() + else: + ports = self.__ports(ports) + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # check async barrier + if not type(async_barrier) is bool: + raise STLArgumentError('async_barrier', async_barrier) + + + # if the user requested a barrier - use it + if async_barrier: + rc = self.async_client.barrier() + if not rc: + raise STLError(rc) + + return self.__get_stats(ports) + + # return all async events + def get_events (self): + return self.event_handler.get_events() + + ############################ Commands ############################# + ############################ ############################# + ############################ ############################# + + + """ + Sets verbose level + + :parameters: + level : str + "high" + "low" + "normal" + + :raises: + None + + """ + def set_verbose (self, level): + modes = {'low' : LoggerApi.VERBOSE_QUIET, 'normal': LoggerApi.VERBOSE_REGULAR, 'high': LoggerApi.VERBOSE_HIGH} + + if not level in modes.keys(): + raise STLArgumentError('level', level) + + self.logger.set_verbose(modes[level]) + + + """ + Connects to the TRex server + + :parameters: + None + + :raises: + + :exc:`STLError` + + """ + @__api_check(False) + def connect (self): + rc = self.__connect() + if not rc: + raise STLError(rc) + + + """ + Disconnects from the server + + :parameters: + stop_traffic : bool + tries to stop traffic before disconnecting + release_ports : bool + tries to release all the acquired ports + + """ + @__api_check(False) + def disconnect (self, stop_traffic = True, release_ports = True): + + # try to stop ports but do nothing if not possible + if stop_traffic: + try: + self.stop() + except STLError: + pass + + + self.logger.pre_cmd("Disconnecting from server at '{0}':'{1}'".format(self.connection_info['server'], + self.connection_info['sync_port'])) + rc = self.__disconnect(release_ports) + self.logger.post_cmd(rc) + + + + """ + Acquires ports for executing commands + + :parameters: + ports : list + ports to execute the command + force : bool + force acquire the ports + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def acquire (self, ports = None, force = False): + # by default use all ports + if ports == None: + ports = self.get_all_ports() + + # verify ports + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # verify valid port id list + if force: + self.logger.pre_cmd("Force acquiring ports {0}:".format(ports)) + else: + self.logger.pre_cmd("Acquiring ports {0}:".format(ports)) + + rc = self.__acquire(ports, force) + + self.logger.post_cmd(rc) + + if not rc: + # cleanup + self.__release(ports) + raise STLError(rc) + + + """ + Release ports + + :parameters: + ports : list + ports to execute the command + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def release (self, ports = None): + # by default use all acquired ports + if ports == None: + ports = self.get_acquired_ports() + + # verify ports + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.logger.pre_cmd("Releasing ports {0}:".format(ports)) + rc = self.__release(ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + """ + Pings the server + + :parameters: + None + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def ping(self): + self.logger.pre_cmd( "Pinging the server on '{0}' port '{1}': ".format(self.connection_info['server'], + self.connection_info['sync_port'])) + rc = self._transmit("ping") + + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + + """ + force acquire ports, stop the traffic, remove all streams and clear stats + + :parameters: + ports : list + ports to execute the command + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def reset(self, ports = None): + + # by default use all ports + if ports == None: + ports = self.get_all_ports() + + # verify ports + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.acquire(ports, force = True) + self.stop(ports) + self.remove_all_streams(ports) + self.clear_stats(ports) + + + """ + remove all streams from port(s) + + :parameters: + ports : list + ports to execute the command + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def remove_all_streams (self, ports = None): + + # by default use all ports + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.logger.pre_cmd("Removing all streams from port(s) {0}:".format(ports)) + rc = self.__remove_all_streams(ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + """ + add a list of streams to port(s) + + :parameters: + ports : list + ports to execute the command + streams: list + streams to attach + + :returns: + list of stream IDs in order of the stream list + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def add_streams (self, streams, ports = None): + # by default use all ports + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # transform single stream + if not isinstance(streams, list): + streams = [streams] + + # check streams + if not all([isinstance(stream, STLStream) for stream in streams]): + raise STLArgumentError('streams', streams) + + self.logger.pre_cmd("Attaching {0} streams to port(s) {1}:".format(len(streams), ports)) + rc = self.__add_streams(streams, ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + return [stream.get_id() for stream in streams] + + + """ + remove a list of streams from ports + + :parameters: + ports : list + ports to execute the command + stream_id_list: list + stream id list to remove + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def remove_streams (self, stream_id_list, ports = None): + # by default use all ports + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # transform single stream + if not isinstance(stream_id_list, list): + stream_id_list = [stream_id_list] + + # check streams + if not all([isinstance(stream_id, long) for stream_id in stream_id_list]): + raise STLArgumentError('stream_id_list', stream_id_list) + + # remove streams + self.logger.pre_cmd("Removing {0} streams from port(s) {1}:".format(len(stream_id_list), ports)) + rc = self.__remove_streams(stream_id_list, ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + """ + load a profile from file + + :parameters: + filename : str + filename to load + + :returns: + list of streams from the profile + + :raises: + + :exc:`STLError` + + """ + @staticmethod + def load_profile (filename): + + # check filename + if not os.path.isfile(filename): + raise STLError("file '{0}' does not exists".format(filename)) + + streams = None + + # try YAML + try: + streams = STLStream.load_from_yaml(filename) + print "***** YAML IS NOT WORKING !!! *********" + + + except YAMLError: + # try python loader + try: + basedir = os.path.dirname(filename) + + sys.path.append(basedir) + file = os.path.basename(filename).split('.')[0] + module = __import__(file, globals(), locals(), [], -1) + reload(module) # reload the update + + streams = module.register().get_streams() + + except Exception as e : + print str(e); + traceback.print_exc(file=sys.stdout) + raise STLError("Unexpected error: '{0}'".format(filename)) + + return streams + + + + + """ + start traffic on port(s) + + :parameters: + ports : list + ports to execute command + + mult : str + multiplier in a form of pps, bps, or line util in % + examples: "5kpps", "10gbps", "85%", "32mbps" + + force : bool + imply stopping the port of active and also + forces a profile that exceeds the L1 BW + + duration : int + limit the run for time in seconds + -1 means unlimited + + total : bool + should the B/W be divided by the ports + or duplicated for each + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def start (self, + ports = None, + mult = "1", + force = False, + duration = -1, + total = False): + + + # by default use all ports + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # verify multiplier + mult_obj = parsing_opts.decode_multiplier(mult, + allow_update = False, + divide_count = len(ports) if total else 1) + if not mult_obj: + raise STLArgumentError('mult', mult) + + # some type checkings + + if not type(force) is bool: + raise STLArgumentError('force', force) + + if not isinstance(duration, (int, float)): + raise STLArgumentError('duration', duration) + + if not type(total) is bool: + raise STLArgumentError('total', total) + + + # verify ports are stopped or force stop them + active_ports = list(set(self.get_active_ports()).intersection(ports)) + if active_ports: + if not force: + raise STLError("Port(s) {0} are active - please stop them or specify 'force'".format(active_ports)) + else: + rc = self.stop(active_ports) + if not rc: + raise STLError(rc) + + + # start traffic + self.logger.pre_cmd("Starting traffic on port(s) {0}:".format(ports)) + rc = self.__start(mult_obj, duration, ports, force) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + + + """ + stop port(s) + + :parameters: + ports : list + ports to execute the command + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def stop (self, ports = None): + + # by default the user means all the active ports + if ports == None: + ports = self.get_active_ports() + if not ports: + return + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.logger.pre_cmd("Stopping traffic on port(s) {0}:".format(ports)) + rc = self.__stop(ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + + """ + update traffic on port(s) + + :parameters: + ports : list + ports to execute command + + mult : str + multiplier in a form of pps, bps, or line util in % + and also with +/- + examples: "5kpps+", "10gbps-", "85%", "32mbps", "20%+" + + force : bool + forces a profile that exceeds the L1 BW + + total : bool + should the B/W be divided by the ports + or duplicated for each + + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def update (self, ports = None, mult = "1", total = False, force = False): + + # by default the user means all the active ports + if ports == None: + ports = self.get_active_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # verify multiplier + mult_obj = parsing_opts.decode_multiplier(mult, + allow_update = True, + divide_count = len(ports) if total else 1) + if not mult_obj: + raise STLArgumentError('mult', mult) + + # verify total + if not type(total) is bool: + raise STLArgumentError('total', total) + + + # call low level functions + self.logger.pre_cmd("Updating traffic on port(s) {0}:".format(ports)) + rc = self.__update(mult, ports, force) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + + """ + pause traffic on port(s) + + :parameters: + ports : list + ports to execute command + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def pause (self, ports = None): + + # by default the user means all the TX ports + if ports == None: + ports = self.get_transmitting_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.logger.pre_cmd("Pausing traffic on port(s) {0}:".format(ports)) + rc = self.__pause(ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + + """ + resume traffic on port(s) + + :parameters: + ports : list + ports to execute command + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def resume (self, ports = None): + + # by default the user means all the paused ports + if ports == None: + ports = self.get_paused_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + self.logger.pre_cmd("Resume traffic on port(s) {0}:".format(ports)) + rc = self.__resume(ports) + self.logger.post_cmd(rc) + + if not rc: + raise STLError(rc) + + + """ + validate port(s) configuration + + :parameters: + ports : list + ports to execute command + + mult : str + multiplier in a form of pps, bps, or line util in % + examples: "5kpps", "10gbps", "85%", "32mbps" + + duration : int + limit the run for time in seconds + -1 means unlimited + + total : bool + should the B/W be divided by the ports + or duplicated for each + + :raises: + + :exc:`STLError` + + """ + @__api_check(True) + def validate (self, ports = None, mult = "1", duration = "-1", total = False): + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # verify multiplier + mult_obj = parsing_opts.decode_multiplier(mult, + allow_update = True, + divide_count = len(ports) if total else 1) + if not mult_obj: + raise STLArgumentError('mult', mult) + + + if not isinstance(duration, (int, float)): + raise STLArgumentError('duration', duration) + + + self.logger.pre_cmd("Validating streams on port(s) {0}:".format(ports)) + rc = self.__validate(ports) + self.logger.post_cmd(rc) + + + for port in ports: + self.ports[port].print_profile(mult_obj, duration) + + + """ + clear stats on port(s) + + :parameters: + ports : list + ports to execute command + + clear_global : bool + clear the global stats + + :raises: + + :exc:`STLError` + + """ + @__api_check(False) + def clear_stats (self, ports = None, clear_global = True): + + # by default use all ports + if ports == None: + ports = self.get_all_ports() + else: + ports = self.__ports(ports) + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + # verify clear global + if not type(clear_global) is bool: + raise STLArgumentError('clear_global', clear_global) + + + rc = self.__clear_stats(ports, clear_global) + if not rc: + raise STLError(rc) + + + + + + """ + block until specify port(s) traffic has ended + + :parameters: + ports : list + ports to execute command + + timeout : int + timeout in seconds + + :raises: + + :exc:`STLTimeoutError` - in case timeout has expired + + :exe:'STLError' + + """ + @__api_check(True) + def wait_on_traffic (self, ports = None, timeout = 60): + + # by default use all acquired ports + if ports == None: + ports = self.get_acquired_ports() + + # verify valid port id list + rc = self._validate_port_list(ports) + if not rc: + raise STLArgumentError('ports', ports, valid_values = self.get_all_ports()) + + expr = time.time() + timeout + + # wait while any of the required ports are active + while set(self.get_active_ports()).intersection(ports): + time.sleep(0.01) + if time.time() > expr: + raise STLTimeoutError(timeout) + + + """ + clear all events + + :parameters: + None + + :raises: + None + + """ + def clear_events (self): + self.event_handler.clear_events() + + + ############################ Line ############################# + ############################ Commands ############################# + ############################ ############################# + + # console decorator + def __console(f): + def wrap(*args): + client = args[0] + + time1 = time.time() + + try: + rc = f(*args) + except STLError as e: + client.logger.log("Log:\n" + format_text(e.brief() + "\n", 'bold')) + return + + # if got true - print time + if rc: + delta = time.time() - time1 + client.logger.log(format_time(delta) + "\n") + + + return wrap + + + @__console + def connect_line (self, line): + '''Connects to the TRex server''' + # define a parser + parser = parsing_opts.gen_parser(self, + "connect", + self.connect_line.__doc__, + parsing_opts.FORCE) + + opts = parser.parse_args(line.split()) + + if opts is None: + return + + # call the API + self.connect() + self.acquire(force = opts.force) + + # true means print time + return True + + @__console + def disconnect_line (self, line): + self.disconnect() + + + + @__console + def reset_line (self, line): + self.reset() + + # true means print time + return True + + + @__console + def start_line (self, line): + '''Start selected traffic in specified ports on TRex\n''' + # define a parser + parser = parsing_opts.gen_parser(self, + "start", + self.start_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.TOTAL, + parsing_opts.FORCE, + parsing_opts.STREAM_FROM_PATH_OR_FILE, + parsing_opts.DURATION, + parsing_opts.MULTIPLIER_STRICT, + parsing_opts.DRY_RUN) + + opts = parser.parse_args(line.split()) + + + if opts is None: + return + + + active_ports = list(set(self.get_active_ports()).intersection(opts.ports)) + + if active_ports: + if not opts.force: + msg = "Port(s) {0} are active - please stop them or add '--force'\n".format(active_ports) + self.logger.log(format_text(msg, 'bold')) + return + else: + self.stop(active_ports) + + + # remove all streams + self.remove_all_streams(opts.ports) + + # pack the profile + streams = self.load_profile(opts.file[0]) + self.add_streams(streams, ports = opts.ports) + + if opts.dry: + self.validate(opts.ports, opts.mult, opts.duration, opts.total) + else: + self.start(opts.ports, + opts.mult, + opts.force, + opts.duration, + opts.total) + + # true means print time + return True + + + + @__console + def stop_line (self, line): + '''Stop active traffic in specified ports on TRex\n''' + parser = parsing_opts.gen_parser(self, + "stop", + self.stop_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find the relevant ports + ports = list(set(self.get_active_ports()).intersection(opts.ports)) + + if not ports: + self.logger.log(format_text("No active traffic on provided ports\n", 'bold')) + return + + self.stop(ports) + + # true means print time + return True + + + @__console + def update_line (self, line): + '''Update port(s) speed currently active\n''' + parser = parsing_opts.gen_parser(self, + "update", + self.update_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.MULTIPLIER, + parsing_opts.TOTAL, + parsing_opts.FORCE) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find the relevant ports + ports = list(set(self.get_active_ports()).intersection(opts.ports)) + + if not ports: + self.logger.log(format_text("No ports in valid state to update\n", 'bold')) + return + + self.update(ports, opts.mult, opts.total, opts.force) + + # true means print time + return True + + + @__console + def pause_line (self, line): + '''Pause active traffic in specified ports on TRex\n''' + parser = parsing_opts.gen_parser(self, + "pause", + self.pause_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find the relevant ports + ports = list(set(self.get_transmitting_ports()).intersection(opts.ports)) + + if not ports: + self.logger.log(format_text("No ports in valid state to pause\n", 'bold')) + return + + self.pause(ports) + + # true means print time + return True + + + @__console + def resume_line (self, line): + '''Resume active traffic in specified ports on TRex\n''' + parser = parsing_opts.gen_parser(self, + "resume", + self.resume_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + # find the relevant ports + ports = list(set(self.get_paused_ports()).intersection(opts.ports)) + + if not ports: + self.logger.log(format_text("No ports in valid state to resume\n", 'bold')) + return + + return self.resume(ports) + + # true means print time + return True + + + @__console + def clear_stats_line (self, line): + '''Clear cached local statistics\n''' + # define a parser + parser = parsing_opts.gen_parser(self, + "clear", + self.clear_stats_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split()) + + if opts is None: + return + + self.clear_stats(opts.ports) + + + + + @__console + def show_stats_line (self, line): + '''Fetch statistics from TRex server by port\n''' + # define a parser + parser = parsing_opts.gen_parser(self, + "stats", + self.show_stats_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.STATS_MASK) + + opts = parser.parse_args(line.split()) + + if opts is None: + return + + # determine stats mask + mask = self.__get_mask_keys(**self.__filter_namespace_args(opts, trex_stl_stats.ALL_STATS_OPTS)) + if not mask: + # set to show all stats if no filter was given + mask = trex_stl_stats.ALL_STATS_OPTS + + stats_opts = trex_stl_stats.ALL_STATS_OPTS.intersection(mask) + + stats = self._get_formatted_stats(opts.ports, mask) + + + # print stats to screen + for stat_type, stat_data in stats.iteritems(): + text_tables.print_table_with_header(stat_data.text_table, stat_type) + + + @__console + def show_streams_line(self, line): + '''Fetch streams statistics from TRex server by port\n''' + # define a parser + parser = parsing_opts.gen_parser(self, + "streams", + self.show_streams_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL, + parsing_opts.STREAMS_MASK) + + opts = parser.parse_args(line.split()) + + if opts is None: + return + + streams = self._get_streams(opts.ports, set(opts.streams)) + if not streams: + self.logger.log(format_text("No streams found with desired filter.\n", "bold", "magenta")) + + else: + # print stats to screen + for stream_hdr, port_streams_data in streams.iteritems(): + text_tables.print_table_with_header(port_streams_data.text_table, + header= stream_hdr.split(":")[0] + ":", + untouched_header= stream_hdr.split(":")[1]) + + + + + @__console + def validate_line (self, line): + '''validates port(s) stream configuration\n''' + + parser = parsing_opts.gen_parser(self, + "validate", + self.validate_line.__doc__, + parsing_opts.PORT_LIST_WITH_ALL) + + opts = parser.parse_args(line.split()) + if opts is None: + return + + self.validate(opts.ports) + + + + \ No newline at end of file diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py b/scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py new file mode 100644 index 00000000..d5b3885d --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_exceptions.py @@ -0,0 +1,54 @@ +import os +import sys + +from trex_control_plane.common.text_opts import * + +# basic error for API +class STLError(Exception): + def __init__ (self, msg): + self.msg = str(msg) + + def __str__ (self): + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + + + s = "\n******\n" + s += "Error at {0}:{1}\n\n".format(format_text(fname, 'bold'), format_text(exc_tb.tb_lineno), 'bold') + s += "specific error:\n\n{0}\n".format(format_text(self.msg, 'bold')) + + return s + + def brief (self): + return self.msg + + +# raised when the client state is invalid for operation +class STLStateError(STLError): + def __init__ (self, op, state): + self.msg = "Operation '{0}' is not valid while '{1}'".format(op, state) + + +# port state error +class STLPortStateError(STLError): + def __init__ (self, port, op, state): + self.msg = "Operation '{0}' on port(s) '{1}' is not valid while port(s) '{2}'".format(op, port, state) + + +# raised when argument is not valid for operation +class STLArgumentError(STLError): + def __init__ (self, name, got, valid_values = None, extended = None): + self.msg = "Argument: '{0}' invalid value: '{1}'".format(name, got) + if valid_values: + self.msg += " - valid values are '{0}'".format(valid_values) + + if extended: + self.msg += "\n{0}".format(extended) + +# raised when timeout occurs +class STLTimeoutError(STLError): + def __init__ (self, timeout): + self.msg = "Timeout: operation took more than '{0}' seconds".format(timeout) + + + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py new file mode 100644 index 00000000..887681a7 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_jsonrpc_client.py @@ -0,0 +1,243 @@ +#!/router/bin/python + +import zmq +import json +import client_utils.general_utils +import re +from time import sleep +from collections import namedtuple +from trex_stl_types import * + +class bcolors: + BLUE = '\033[94m' + GREEN = '\033[32m' + YELLOW = '\033[93m' + RED = '\033[31m' + MAGENTA = '\033[35m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +# sub class to describe a batch +class BatchMessage(object): + def __init__ (self, rpc_client): + self.rpc_client = rpc_client + self.batch_list = [] + + def add (self, method_name, params={}): + + id, msg = self.rpc_client.create_jsonrpc_v2(method_name, params, encode = False) + self.batch_list.append(msg) + + def invoke(self, block = False): + if not self.rpc_client.connected: + return RC_ERR("Not connected to server") + + msg = json.dumps(self.batch_list) + + return self.rpc_client.send_raw_msg(msg) + + +# JSON RPC v2.0 client +class JsonRpcClient(object): + + def __init__ (self, default_server, default_port, logger): + self.logger = logger + self.connected = False + + # default values + self.port = default_port + self.server = default_server + self.id_gen = client_utils.general_utils.random_id_gen() + + + def get_connection_details (self): + rc = {} + rc['server'] = self.server + rc['port'] = self.port + + return rc + + # pretty print for JSON + def pretty_json (self, json_str, use_colors = True): + pretty_str = json.dumps(json.loads(json_str), indent = 4, separators=(',', ': '), sort_keys = True) + + if not use_colors: + return pretty_str + + try: + # int numbers + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*[^.])',r'\1{0}\2{1}'.format(bcolors.BLUE, bcolors.ENDC), pretty_str) + # float + pretty_str = re.sub(r'([ ]*:[ ]+)(\-?[1-9][0-9]*\.[0-9]+)',r'\1{0}\2{1}'.format(bcolors.MAGENTA, bcolors.ENDC), pretty_str) + # strings + + pretty_str = re.sub(r'([ ]*:[ ]+)("[^"]*")',r'\1{0}\2{1}'.format(bcolors.RED, bcolors.ENDC), pretty_str) + pretty_str = re.sub(r"('[^']*')", r'{0}\1{1}'.format(bcolors.MAGENTA, bcolors.RED), pretty_str) + except : + pass + + return pretty_str + + def verbose_msg (self, msg): + self.logger.log("\n\n[verbose] " + msg, level = self.logger.VERBOSE_HIGH) + + + # batch messages + def create_batch (self): + return BatchMessage(self) + + def create_jsonrpc_v2 (self, method_name, params = {}, encode = True): + msg = {} + msg["jsonrpc"] = "2.0" + msg["method"] = method_name + + msg["params"] = params + + msg["id"] = self.id_gen.next() + + if encode: + return id, json.dumps(msg) + else: + return id, msg + + + def invoke_rpc_method (self, method_name, params = {}): + if not self.connected: + return RC_ERR("Not connected to server") + + id, msg = self.create_jsonrpc_v2(method_name, params) + + return self.send_raw_msg(msg) + + + # low level send of string message + def send_raw_msg (self, msg): + + self.verbose_msg("Sending Request To Server:\n\n" + self.pretty_json(msg) + "\n") + + tries = 0 + while True: + try: + self.socket.send(msg) + break + except zmq.Again: + tries += 1 + if tries > 5: + self.disconnect() + return RC_ERR("*** [RPC] - Failed to send message to server") + + + tries = 0 + while True: + try: + response = self.socket.recv() + break + except zmq.Again: + tries += 1 + if tries > 5: + self.disconnect() + return RC_ERR("*** [RPC] - Failed to get server response at {0}".format(self.transport)) + + + self.verbose_msg("Server Response:\n\n" + self.pretty_json(response) + "\n") + + # decode + + # batch ? + response_json = json.loads(response) + + if isinstance(response_json, list): + rc_batch = RC() + + for single_response in response_json: + rc = self.process_single_response(single_response) + rc_batch.add(rc) + + return rc_batch + + else: + return self.process_single_response(response_json) + + + def process_single_response (self, response_json): + + if (response_json.get("jsonrpc") != "2.0"): + return RC_ERR("Malformed Response ({0})".format(str(response_json))) + + # error reported by server + if ("error" in response_json): + if "specific_err" in response_json["error"]: + return RC_ERR(response_json["error"]["specific_err"]) + else: + return RC_ERR(response_json["error"]["message"]) + + + # if no error there should be a result + if ("result" not in response_json): + return RC_ERR("Malformed Response ({0})".format(str(response_json))) + + return RC_OK(response_json["result"]) + + + + def disconnect (self): + if self.connected: + self.socket.close(linger = 0) + self.context.destroy(linger = 0) + self.connected = False + return RC_OK() + else: + return RC_ERR("Not connected to server") + + + def connect(self, server = None, port = None): + if self.connected: + self.disconnect() + + self.context = zmq.Context() + + self.server = (server if server else self.server) + self.port = (port if port else self.port) + + # Socket to talk to server + self.transport = "tcp://{0}:{1}".format(self.server, self.port) + + self.socket = self.context.socket(zmq.REQ) + try: + self.socket.connect(self.transport) + except zmq.error.ZMQError as e: + return RC_ERR("ZMQ Error: Bad server or port name: " + str(e)) + + self.socket.setsockopt(zmq.SNDTIMEO, 1000) + self.socket.setsockopt(zmq.RCVTIMEO, 1000) + + self.connected = True + + rc = self.invoke_rpc_method('ping') + if not rc: + self.connected = False + return rc + + return RC_OK() + + + def reconnect(self): + # connect using current values + return self.connect() + + if not self.connected: + return RC_ERR("Not connected to server") + + # reconnect + return self.connect(self.server, self.port) + + + def is_connected(self): + return self.connected + + def __del__(self): + self.logger.log("Shutting down RPC client\n") + if hasattr(self, "context"): + self.context.destroy(linger=0) + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py b/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py new file mode 100644 index 00000000..b6e7c026 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_interface.py @@ -0,0 +1,43 @@ + +# base object class for a packet builder +class CTrexPktBuilderInterface(object): + + def compile (self): + """ + Compiles the packet and VM + """ + raise Exception("implement me") + + + def dump_pkt(self): + """ + Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) + + :parameters: + None + + :return: + + packet representation as array of bytes + + :raises: + + :exc:`CTRexPktBuilder.EmptyPacketError`, in case packet is empty. + + """ + + raise Exception("implement me") + + + def get_vm_data(self): + """ + Dumps the instructions + + :parameters: + None + + :return: + + json object of instructions + + """ + + raise Exception("implement me") + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py new file mode 100644 index 00000000..8d2d6b8f --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_packet_builder_scapy.py @@ -0,0 +1,747 @@ +import random +import string +import struct +import socket +import json +import yaml +import binascii +import base64 + +from trex_stl_packet_builder_interface import CTrexPktBuilderInterface + +from scapy.all import * + + + +class CTRexPacketBuildException(Exception): + """ + This is the general Packet Building error exception class. + """ + def __init__(self, code, message): + self.code = code + self.message = message + + def __str__(self): + return self.__repr__() + + def __repr__(self): + return u"[errcode:%r] %r" % (self.code, self.message) + +################################################################################################ + +def ipv4_str_to_num (ipv4_buffer): + + assert type(ipv4_buffer)==str, 'type of ipv4_buffer is not str' + assert len(ipv4_buffer)==4, 'size of ipv4_buffer is not 4' + res=0 + shift=24 + for i in ipv4_buffer: + res = res + (ord(i)< max_value: + raise CTRexPacketBuildException(-12, 'min is greater than max'); + if min_value == max_value: + raise CTRexPacketBuildException(-13, "min value is equal to max value, you can't use this type of range"); + + +class CTRexScIpv4SimpleRange(CTRexScFieldRangeBase): + """ + range of ipv4 ip + """ + def __init__(self, field_name, field_type, min_ip, max_ip): + super(CTRexScIpv4SimpleRange, self).__init__(field_name,field_type) + self.min_ip = min_ip + self.max_ip = max_ip + mmin=ipv4_str_to_num (is_valid_ipv4(min_ip)) + mmax=ipv4_str_to_num (is_valid_ipv4(max_ip)) + if mmin > mmax : + raise CTRexPacketBuildException(-11, 'CTRexScIpv4SimpleRange m_min ip is bigger than max'); + + +class CTRexScIpv4TupleGen(CTRexScriptsBase): + """ + range tuple + """ + FLAGS_ULIMIT_FLOWS =1 + + def __init__(self, min_ipv4, max_ipv4, num_flows=100000, min_port=1025, max_port=65535, flags=0): + super(CTRexScIpv4TupleGen, self).__init__() + self.min_ip = min_ipv4 + self.max_ip = max_ipv4 + mmin=ipv4_str_to_num (is_valid_ipv4(min_ipv4)) + mmax=ipv4_str_to_num (is_valid_ipv4(max_ipv4)) + if mmin > mmax : + raise CTRexPacketBuildException(-11, 'CTRexScIpv4SimpleRange m_min ip is bigger than max'); + + self.num_flows=num_flows; + + self.min_port =min_port + self.max_port =max_port + self.flags = flags + + +class CTRexScTrimPacketSize(CTRexScriptsBase): + """ + trim packet size. field type is CTRexScFieldRangeBase.FILED_TYPES = ["inc","dec","rand"] + """ + def __init__(self,field_type="rand",min_pkt_size=None, max_pkt_size=None): + super(CTRexScTrimPacketSize, self).__init__() + self.field_type = field_type + self.min_pkt_size = min_pkt_size + self.max_pkt_size = max_pkt_size + if max_pkt_size != None and min_pkt_size !=None : + if min_pkt_size == max_pkt_size: + raise CTRexPacketBuildException(-11, 'CTRexScTrimPacketSize min_pkt_size is the same as max_pkt_size '); + + if min_pkt_size > max_pkt_size: + raise CTRexPacketBuildException(-11, 'CTRexScTrimPacketSize min_pkt_size is bigger than max_pkt_size '); + +class CTRexScRaw(CTRexScriptsBase): + """ + raw instructions + """ + def __init__(self,list_of_commands=None): + super(CTRexScRaw, self).__init__() + if list_of_commands==None: + self.commands =[] + else: + self.commands = list_of_commands + + def add_cmd (self,cmd): + self.commands.append(cmd) + + + +################################################################################################ +# VM raw instructions +################################################################################################ + +class CTRexVmInsBase(object): + """ + instruction base + """ + def __init__(self, ins_type): + self.type = ins_type + assert type(ins_type)==str, 'type of ins_type is not str' + +class CTRexVmInsFixIpv4(CTRexVmInsBase): + def __init__(self, offset): + super(CTRexVmInsFixIpv4, self).__init__("fix_checksum_ipv4") + self.pkt_offset = offset + assert type(offset)==int, 'type of offset is not int' + + +class CTRexVmInsFlowVar(CTRexVmInsBase): + #TBD add more validation tests + + OPERATIONS =['inc', 'dec', 'random'] + VALID_SIZES =[1, 2, 4, 8] + + def __init__(self, fv_name, size, op, init_value, min_value, max_value): + super(CTRexVmInsFlowVar, self).__init__("flow_var") + self.name = fv_name; + assert type(fv_name)==str, 'type of fv_name is not str' + self.size = size + self.op = op + self.init_value = init_value + assert type(init_value)==int, 'type of init_value is not int' + self.min_value=min_value + assert type(min_value)==int, 'type of min_value is not int' + self.max_value=max_value + assert type(max_value)==int, 'type of min_value is not int' + +class CTRexVmInsWrFlowVar(CTRexVmInsBase): + def __init__(self, fv_name, pkt_offset, add_value=0, is_big_endian=True): + super(CTRexVmInsWrFlowVar, self).__init__("write_flow_var") + self.name = fv_name + assert type(fv_name)==str, 'type of fv_name is not str' + self.pkt_offset = pkt_offset + assert type(pkt_offset)==int, 'type of pkt_offset is not int' + self.add_value = add_value + assert type(add_value)==int, 'type of add_value is not int' + self.is_big_endian = is_big_endian + assert type(is_big_endian)==bool, 'type of is_big_endian is not bool' + +class CTRexVmInsTrimPktSize(CTRexVmInsBase): + def __init__(self,fv_name): + super(CTRexVmInsTrimPktSize, self).__init__("trim_pkt_size") + self.name = fv_name + assert type(fv_name)==str, 'type of fv_name is not str' + +class CTRexVmInsTupleGen(CTRexVmInsBase): + def __init__(self, fv_name, ip_min, ip_max, port_min, port_max, limit_flows, flags=0): + super(CTRexVmInsTupleGen, self).__init__("tuple_flow_var") + self.name =fv_name + assert type(fv_name)==str, 'type of fv_name is not str' + self.ip_min = ip_min; + self.ip_max = ip_max; + self.port_min = port_min; + self.port_max = port_max; + self.limit_flows = limit_flows; + self.flags =flags; + + +################################################################################################ +# +class CTRexVmEngine(object): + + def __init__(self): + """ + inlcude list of instruction + """ + super(CTRexVmEngine, self).__init__() + self.ins=[] + self.split_by_var = '' + + # return as json + def get_json (self): + inst_array = []; + # dump it as dict + for obj in self.ins: + inst_array.append(obj.__dict__); + + return {'instructions': inst_array, 'split_by_var': self.split_by_var} + + def add_ins (self,ins): + #assert issubclass(ins, CTRexVmInsBase) + self.ins.append(ins); + + def dump (self): + cnt=0; + for obj in self.ins: + print "ins",cnt + cnt = cnt +1 + print obj.__dict__ + + def dump_bjson (self): + print json.dumps(self.get_json(), sort_keys=True, indent=4) + + def dump_as_yaml (self): + print yaml.dump(self.get_json(), default_flow_style=False) + + + +################################################################################################ + +class CTRexScapyPktUtl(object): + + def __init__(self, scapy_pkt): + self.pkt = scapy_pkt + + def pkt_iter (self): + p=self.pkt; + while True: + yield p + p=p.payload + if p ==None or isinstance(p,NoPayload): + break; + + def get_list_iter(self): + l=list(self.pkt_iter()) + return l + + + def get_pkt_layers(self): + """ + return string 'IP:UDP:TCP' + """ + l=self.get_list_iter (); + l1=map(lambda p: p.name,l ); + return ":".join(l1); + + def _layer_offset(self, name, cnt = 0): + """ + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + """ + save_cnt=cnt + for pkt in self.pkt_iter (): + if pkt.name == name: + if cnt==0: + return (pkt, pkt.offset) + else: + cnt=cnt -1 + + raise CTRexPacketBuildException(-11,("no layer %s-%d" % (name, save_cnt))); + + + def layer_offset(self, name, cnt = 0): + """ + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + """ + save_cnt=cnt + for pkt in self.pkt_iter (): + if pkt.name == name: + if cnt==0: + return pkt.offset + else: + cnt=cnt -1 + + raise CTRexPacketBuildException(-11,("no layer %s-%d" % (name, save_cnt))); + + def get_field_offet(self, layer, layer_cnt, field_name): + """ + return offset of layer e.g 'IP',1 will return offfset of layer ip:1 + """ + t=self._layer_offset(layer,layer_cnt); + l_offset=t[1]; + layer_pkt=t[0] + + #layer_pkt.dump_fields_offsets () + + for f in layer_pkt.fields_desc: + if f.name == field_name: + return (l_offset+f.offset,f.get_size_bytes ()); + + raise CTRexPacketBuildException(-11, "no layer %s-%d." % (name, save_cnt, field_name)); + + def get_layer_offet_by_str(self, layer_des): + """ + return layer offset by string + + :parameters: + + IP:0 + IP:1 + return offset + + + """ + l1=layer_des.split(":") + layer="" + layer_cnt=0; + + if len(l1)==1: + layer=l1[0]; + else: + layer=l1[0]; + layer_cnt=int(l1[1]); + + return self.layer_offset(layer, layer_cnt) + + + + def get_field_offet_by_str(self, field_des): + """ + return field_des (offset,size) layer:cnt.field + for example + 802|1Q.vlan get 802.1Q->valn replace | with . + IP.src + IP:0.src (first IP.src like IP.src) + for example IP:1.src for internal IP + + return (offset, size) as tuple + + + """ + + s=field_des.split("."); + if len(s)!=2: + raise CTRexPacketBuildException(-11, ("field desription should be layer:cnt.field e.g IP.src or IP:1.src")); + + + layer_ex = s[0].replace("|",".") + field = s[1] + + l1=layer_ex.split(":") + layer="" + layer_cnt=0; + + if len(l1)==1: + layer=l1[0]; + else: + layer=l1[0]; + layer_cnt=int(l1[1]); + + return self.get_field_offet(layer,layer_cnt,field) + + def has_IPv4 (self): + return self.pkt.has_layer("IP"); + + def has_IPv6 (self): + return self.pkt.has_layer("IPv6"); + + def has_UDP (self): + return self.pkt.has_layer("UDP"); + +################################################################################################ + +class CTRexVmDescBase(object): + """ + instruction base + """ + def __init__(self): + pass; + + def get_obj(self): + return self; + + def get_json(self): + return self.get_obj().__dict__ + + def dump_bjson(self): + print json.dumps(self.get_json(), sort_keys=True, indent=4) + + def dump_as_yaml(self): + print yaml.dump(self.get_json(), default_flow_style=False) + + + def get_var_ref (self): + ''' + virtual function return a ref var name + ''' + return None + + def get_var_name(self): + ''' + virtual function return the varible name if exists + ''' + return None + + def compile(self,parent): + ''' + virtual function to take parent than has function name_to_offset + ''' + pass; + + +def valid_fv_size (size): + if not (size in CTRexVmInsFlowVar.VALID_SIZES): + raise CTRexPacketBuildException(-11,("flow var has not valid size %d ") % size ); + +def valid_fv_ops (op): + if not (op in CTRexVmInsFlowVar.OPERATIONS): + raise CTRexPacketBuildException(-11,("flow var does not have a valid op %s ") % op ); + +def convert_val (val): + if type(val) == int: + return val + else: + if type(val) == str: + return ipv4_str_to_num (is_valid_ipv4(val)) + else: + raise CTRexPacketBuildException(-11,("init val not valid %s ") % val ); + +def check_for_int (val): + assert type(val)==int, 'type of vcal is not int' + + +class CTRexVmDescFlowVar(CTRexVmDescBase): + def __init__(self, name, init_value=None, min_value=0, max_value=255, size=4, op="inc"): + super(CTRexVmDescFlowVar, self).__init__() + self.name = name; + assert type(name)==str, 'type of name is not str' + self.size =size + valid_fv_size(size) + self.op =op + valid_fv_ops (op) + + # choose default value for init val + if init_value == None: + init_value = max_value if op == "dec" else min_value + + self.init_value = convert_val (init_value) + self.min_value = convert_val (min_value); + self.max_value = convert_val (max_value) + + if self.min_value > self.max_value : + raise CTRexPacketBuildException(-11,("max %d is lower than min %d ") % (self.max_value,self.min_value) ); + + def get_obj (self): + return CTRexVmInsFlowVar(self.name,self.size,self.op,self.init_value,self.min_value,self.max_value); + + def get_var_name(self): + return [self.name] + + +class CTRexVmDescFixIpv4(CTRexVmDescBase): + def __init__(self, offset): + super(CTRexVmDescFixIpv4, self).__init__() + self.offset = offset; # could be a name of offset + + def get_obj (self): + return CTRexVmInsFixIpv4(self.offset); + + def compile(self,parent): + if type(self.offset)==str: + self.offset = parent._pkt_layer_offset(self.offset); + +class CTRexVmDescWrFlowVar(CTRexVmDescBase): + def __init__(self, fv_name, pkt_offset, offset_fixup=0, add_val=0, is_big=True): + super(CTRexVmDescWrFlowVar, self).__init__() + self.name =fv_name + assert type(fv_name)==str, 'type of fv_name is not str' + self.offset_fixup =offset_fixup + assert type(offset_fixup)==int, 'type of offset_fixup is not int' + self.pkt_offset =pkt_offset + self.add_val =add_val + assert type(add_val)==int,'type of add_val is not int' + self.is_big =is_big; + assert type(is_big)==bool,'type of is_big_endian is not bool' + + def get_var_ref (self): + return self.name + + def get_obj (self): + return CTRexVmInsWrFlowVar(self.name,self.pkt_offset+self.offset_fixup,self.add_val,self.is_big) + + def compile(self,parent): + if type(self.pkt_offset)==str: + t=parent._name_to_offset(self.pkt_offset) + self.pkt_offset = t[0] + + +class CTRexVmDescTrimPktSize(CTRexVmDescBase): + def __init__(self,fv_name): + super(CTRexVmDescTrimPktSize, self).__init__() + self.name = fv_name + assert type(fv_name)==str, 'type of fv_name is not str' + + def get_var_ref (self): + return self.name + + def get_obj (self): + return CTRexVmInsTrimPktSize(self.name) + + + +class CTRexVmDescTupleGen(CTRexVmDescBase): + def __init__(self,name, ip_min="0.0.0.1", ip_max="0.0.0.10", port_min=1025, port_max=65535, limit_flows=100000, flags=0): + super(CTRexVmDescTupleGen, self).__init__() + self.name = name + assert type(name)==str, 'type of fv_name is not str' + self.ip_min = convert_val(ip_min); + self.ip_max = convert_val(ip_max); + self.port_min = port_min; + check_for_int (port_min) + self.port_max = port_max; + check_for_int(port_max) + self.limit_flows = limit_flows; + check_for_int(limit_flows) + self.flags =flags; + check_for_int(flags) + + def get_var_name(self): + return [self.name+".ip",self.name+".port"] + + def get_obj (self): + return CTRexVmInsTupleGen(self.name, self.ip_min, self.ip_max, self.port_min, self.port_max, self.limit_flows, self.flags); + + +################################################################################################ + + +class CScapyTRexPktBuilder(CTrexPktBuilderInterface): + + """ + This class defines the TRex API of building a packet using dpkt package. + Using this class the user can also define how TRex will handle the packet by specifying the VM setting. + """ + def __init__(self, pkt = None, vm = None): + """ + Instantiate a CTRexPktBuilder object + + :parameters: + None + + """ + super(CScapyTRexPktBuilder, self).__init__() + + self.pkt = None + self.vm_scripts = [] # list of high level instructions + self.vm_low_level = None + self.metadata="" + + + # process packet + if pkt != None: + if not isinstance(pkt, Packet): + raise CTRexPacketBuildException(-14, "bad value for variable pkt") + self.set_packet(pkt) + + # process VM + if vm != None: + if not isinstance(vm, (CTRexScRaw, list)): + raise CTRexPacketBuildException(-14, "bad value for variable vm") + + self.add_command(vm if isinstance(vm, CTRexScRaw) else CTRexScRaw(vm)) + + + def dump_vm_data_as_yaml(self): + print yaml.dump(self.get_vm_data(), default_flow_style=False) + + def get_vm_data(self): + """ + Dumps the instructions + + :parameters: + None + + :return: + + json object of instructions + + :raises: + + :exc:`AssertionError`, in case VM is not compiled (is None). + """ + + assert self.vm_low_level is not None, 'vm_low_level is None, please use compile()' + + return self.vm_low_level.get_json() + + def dump_pkt(self, encode = True): + """ + Dumps the packet as a decimal array of bytes (each item x gets value between 0-255) + + :parameters: + encode : bool + Encode using base64. (disable for debug) + + Default: **True** + + :return: + + packet representation as array of bytes + + :raises: + + :exc:`AssertionError`, in case packet is empty. + + """ + + assert self.pkt, 'empty packet' + + return {'binary': base64.b64encode(str(self.pkt)) if encode else str(self.pkt), + 'meta': self.metadata} + + def dump_pkt_to_pcap(self, file_path): + wrpcap(file_path, self.pkt) + + def add_command (self, script): + self.vm_scripts.append(script.clone()); + + def dump_scripts (self): + self.vm_low_level.dump_as_yaml() + + def set_packet (self, pkt): + """ + Scapy packet Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/IP()/"A"*10 + """ + self.pkt = pkt; + + + def compile (self): + self.vm_low_level=CTRexVmEngine() + assert self.pkt, 'empty packet' + self.pkt.build(); + + + for sc in self.vm_scripts: + if isinstance(sc, CTRexScRaw): + self._compile_raw(sc) + + #for obj in self.vm_scripts: + # # tuple gen script + # if isinstance(obj, CTRexScIpv4TupleGen) + # self._add_tuple_gen(tuple_gen) + + #################################################### + # private + + def _compile_raw (self,obj): + + # make sure we have varibles once + vars={}; + + # add it add var to dit + for desc in obj.commands: + var_names = desc.get_var_name() + + if var_names : + for var_name in var_names: + if vars.has_key(var_name): + raise CTRexPacketBuildException(-11,("variable %s define twice ") % (var_name) ); + else: + vars[var_name]=1 + + # check that all write exits + for desc in obj.commands: + var_name = desc.get_var_ref() + if var_name : + if not vars.has_key(var_name): + raise CTRexPacketBuildException(-11,("variable %s does not exists ") % (var_name) ); + desc.compile(self); + + for desc in obj.commands: + self.vm_low_level.add_ins(desc.get_obj()); + + + def _pkt_layer_offset (self,layer_name): + assert self.pkt != None, 'empty packet' + p_utl=CTRexScapyPktUtl(self.pkt); + return p_utl.get_layer_offet_by_str(layer_name) + + def _name_to_offset(self,field_name): + assert self.pkt != None, 'empty packet' + p_utl=CTRexScapyPktUtl(self.pkt); + return p_utl.get_field_offet_by_str(field_name) + + def _add_tuple_gen(self,tuple_gen): + + pass; + + + + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_port.py b/scripts/automation/trex_control_plane/stl/trex_stl_port.py new file mode 100644 index 00000000..8923d3d6 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_port.py @@ -0,0 +1,512 @@ + +from collections import namedtuple, OrderedDict + +import trex_stl_stats +from trex_stl_types import * + +StreamOnPort = namedtuple('StreamOnPort', ['compiled_stream', 'metadata']) + +########## utlity ############ +def mult_to_factor (mult, max_bps_l2, max_pps, line_util): + if mult['type'] == 'raw': + return mult['value'] + + if mult['type'] == 'bps': + return mult['value'] / max_bps_l2 + + if mult['type'] == 'pps': + return mult['value'] / max_pps + + if mult['type'] == 'percentage': + return mult['value'] / line_util + + +# describes a single port +class Port(object): + STATE_DOWN = 0 + STATE_IDLE = 1 + STATE_STREAMS = 2 + STATE_TX = 3 + STATE_PAUSE = 4 + PortState = namedtuple('PortState', ['state_id', 'state_name']) + STATES_MAP = {STATE_DOWN: "DOWN", + STATE_IDLE: "IDLE", + STATE_STREAMS: "IDLE", + STATE_TX: "ACTIVE", + STATE_PAUSE: "PAUSE"} + + + def __init__ (self, port_id, speed, driver, user, comm_link, session_id): + self.port_id = port_id + self.state = self.STATE_IDLE + self.handler = None + self.comm_link = comm_link + self.transmit = comm_link.transmit + self.transmit_batch = comm_link.transmit_batch + self.user = user + self.driver = driver + self.speed = speed + self.streams = {} + self.profile = None + self.session_id = session_id + + self.port_stats = trex_stl_stats.CPortStats(self) + + + def err(self, msg): + return RC_ERR("port {0} : {1}".format(self.port_id, msg)) + + def ok(self, data = ""): + return RC_OK(data) + + def get_speed_bps (self): + return (self.speed * 1000 * 1000 * 1000) + + # take the port + def acquire(self, force = False): + params = {"port_id": self.port_id, + "user": self.user, + "session_id": self.session_id, + "force": force} + + command = RpcCmdData("acquire", params) + rc = self.transmit(command.method, command.params) + if rc.good(): + self.handler = rc.data() + return self.ok() + else: + return self.err(rc.err()) + + # release the port + def release(self): + params = {"port_id": self.port_id, + "handler": self.handler} + + command = RpcCmdData("release", params) + rc = self.transmit(command.method, command.params) + self.handler = None + + if rc.good(): + return self.ok() + else: + return self.err(rc.err()) + + def is_acquired(self): + return (self.handler != None) + + def is_active(self): + return(self.state == self.STATE_TX ) or (self.state == self.STATE_PAUSE) + + def is_transmitting (self): + return (self.state == self.STATE_TX) + + def is_paused (self): + return (self.state == self.STATE_PAUSE) + + + def sync(self): + params = {"port_id": self.port_id} + + command = RpcCmdData("get_port_status", params) + rc = self.transmit(command.method, command.params) + if rc.bad(): + return self.err(rc.err()) + + # sync the port + port_state = rc.data()['state'] + + if port_state == "DOWN": + self.state = self.STATE_DOWN + elif port_state == "IDLE": + self.state = self.STATE_IDLE + elif port_state == "STREAMS": + self.state = self.STATE_STREAMS + elif port_state == "TX": + self.state = self.STATE_TX + elif port_state == "PAUSE": + self.state = self.STATE_PAUSE + else: + raise Exception("port {0}: bad state received from server '{1}'".format(self.port_id, port_state)) + + # TODO: handle syncing the streams into stream_db + + return self.ok() + + + # return TRUE if write commands + def is_port_writable (self): + # operations on port can be done on state idle or state streams + return ((self.state == self.STATE_IDLE) or (self.state == self.STATE_STREAMS)) + + + # add streams + def add_streams (self, streams_list): + + if not self.is_acquired(): + return self.err("port is not owned") + + if not self.is_port_writable(): + return self.err("Please stop port before attempting to add streams") + + batch = [] + for stream in (streams_list if isinstance(streams_list, list) else [streams_list]): + + params = {"handler": self.handler, + "port_id": self.port_id, + "stream_id": stream.get_id(), + "stream": stream.to_json()} + + cmd = RpcCmdData('add_stream', params) + batch.append(cmd) + + # meta data for show streams + self.streams[stream.get_id()] = StreamOnPort(stream.to_json(), + Port._generate_stream_metadata(stream)) + + rc = self.transmit_batch(batch) + if not rc: + return self.err(rc.err()) + + + + # the only valid state now + self.state = self.STATE_STREAMS + + return self.ok() + + + + # remove stream from port + def remove_streams (self, stream_id_list): + + if not self.is_acquired(): + return self.err("port is not owned") + + if not self.is_port_writable(): + return self.err("Please stop port before attempting to remove streams") + + # single element to list + stream_id_list = stream_id_list if isinstance(stream_id_list, list) else [stream_id_list] + + # verify existance + if not all([stream_id in self.streams for stream_id in stream_id_list]): + return self.err("stream {0} does not exists".format(stream_id)) + + batch = [] + + for stream_id in stream_id_list: + params = {"handler": self.handler, + "port_id": self.port_id, + "stream_id": stream_id} + + cmd = RpcCmdData('remove_stream', params) + batch.append(cmd) + + del self.streams[stream_id] + + + rc = self.transmit_batch(batch) + if not rc: + return self.err(rc.err()) + + self.state = self.STATE_STREAMS if (len(self.streams) > 0) else self.STATE_IDLE + + return self.ok() + + + # remove all the streams + def remove_all_streams (self): + + if not self.is_acquired(): + return self.err("port is not owned") + + if not self.is_port_writable(): + return self.err("Please stop port before attempting to remove streams") + + params = {"handler": self.handler, + "port_id": self.port_id} + + rc = self.transmit("remove_all_streams", params) + if not rc: + return self.err(rc.err()) + + self.streams = {} + + self.state = self.STATE_IDLE + + return self.ok() + + # get a specific stream + def get_stream (self, stream_id): + if stream_id in self.streams: + return self.streams[stream_id] + else: + return None + + def get_all_streams (self): + return self.streams + + # start traffic + def start (self, mul, duration, force): + if not self.is_acquired(): + return self.err("port is not owned") + + if self.state == self.STATE_DOWN: + return self.err("Unable to start traffic - port is down") + + if self.state == self.STATE_IDLE: + return self.err("Unable to start traffic - no streams attached to port") + + if self.state == self.STATE_TX: + return self.err("Unable to start traffic - port is already transmitting") + + params = {"handler": self.handler, + "port_id": self.port_id, + "mul": mul, + "duration": duration, + "force": force} + + rc = self.transmit("start_traffic", params) + if rc.bad(): + return self.err(rc.err()) + + self.state = self.STATE_TX + + return self.ok() + + # stop traffic + # with force ignores the cached state and sends the command + def stop (self, force = False): + + if not self.is_acquired(): + return self.err("port is not owned") + + # port is already stopped + if not force: + if (self.state == self.STATE_IDLE) or (self.state == self.state == self.STATE_STREAMS): + return self.ok() + + + + params = {"handler": self.handler, + "port_id": self.port_id} + + rc = self.transmit("stop_traffic", params) + if rc.bad(): + return self.err(rc.err()) + + # only valid state after stop + self.state = self.STATE_STREAMS + + return self.ok() + + def pause (self): + + if not self.is_acquired(): + return self.err("port is not owned") + + if (self.state != self.STATE_TX) : + return self.err("port is not transmitting") + + params = {"handler": self.handler, + "port_id": self.port_id} + + rc = self.transmit("pause_traffic", params) + if rc.bad(): + return self.err(rc.err()) + + # only valid state after stop + self.state = self.STATE_PAUSE + + return self.ok() + + + def resume (self): + + if not self.is_acquired(): + return self.err("port is not owned") + + if (self.state != self.STATE_PAUSE) : + return self.err("port is not in pause mode") + + params = {"handler": self.handler, + "port_id": self.port_id} + + rc = self.transmit("resume_traffic", params) + if rc.bad(): + return self.err(rc.err()) + + # only valid state after stop + self.state = self.STATE_TX + + return self.ok() + + + def update (self, mul, force): + + if not self.is_acquired(): + return self.err("port is not owned") + + if (self.state != self.STATE_TX) : + return self.err("port is not transmitting") + + params = {"handler": self.handler, + "port_id": self.port_id, + "mul": mul, + "force": force} + + rc = self.transmit("update_traffic", params) + if rc.bad(): + return self.err(rc.err()) + + return self.ok() + + + def validate (self): + + if not self.is_acquired(): + return self.err("port is not owned") + + if (self.state == self.STATE_DOWN): + return self.err("port is down") + + if (self.state == self.STATE_IDLE): + return self.err("no streams attached to port") + + params = {"handler": self.handler, + "port_id": self.port_id} + + rc = self.transmit("validate", params) + if rc.bad(): + return self.err(rc.err()) + + self.profile = rc.data() + + return self.ok() + + def get_profile (self): + return self.profile + + + def print_profile (self, mult, duration): + if not self.get_profile(): + return + + rate = self.get_profile()['rate'] + graph = self.get_profile()['graph'] + + print format_text("Profile Map Per Port\n", 'underline', 'bold') + + factor = mult_to_factor(mult, rate['max_bps_l2'], rate['max_pps'], rate['max_line_util']) + + print "Profile max BPS L2 (base / req): {:^12} / {:^12}".format(format_num(rate['max_bps_l2'], suffix = "bps"), + format_num(rate['max_bps_l2'] * factor, suffix = "bps")) + + print "Profile max BPS L1 (base / req): {:^12} / {:^12}".format(format_num(rate['max_bps_l1'], suffix = "bps"), + format_num(rate['max_bps_l1'] * factor, suffix = "bps")) + + print "Profile max PPS (base / req): {:^12} / {:^12}".format(format_num(rate['max_pps'], suffix = "pps"), + format_num(rate['max_pps'] * factor, suffix = "pps"),) + + print "Profile line util. (base / req): {:^12} / {:^12}".format(format_percentage(rate['max_line_util']), + format_percentage(rate['max_line_util'] * factor)) + + + # duration + exp_time_base_sec = graph['expected_duration'] / (1000 * 1000) + exp_time_factor_sec = exp_time_base_sec / factor + + # user configured a duration + if duration > 0: + if exp_time_factor_sec > 0: + exp_time_factor_sec = min(exp_time_factor_sec, duration) + else: + exp_time_factor_sec = duration + + + print "Duration (base / req): {:^12} / {:^12}".format(format_time(exp_time_base_sec), + format_time(exp_time_factor_sec)) + print "\n" + + + def get_port_state_name(self): + return self.STATES_MAP.get(self.state, "Unknown") + + ################# stats handler ###################### + def generate_port_stats(self): + return self.port_stats.generate_stats() + + def generate_port_status(self): + return {"type": self.driver, + "maximum": "{speed} Gb/s".format(speed=self.speed), + "status": self.get_port_state_name() + } + + def clear_stats(self): + return self.port_stats.clear_stats() + + + def get_stats (self): + return self.port_stats.get_stats() + + + def invalidate_stats(self): + return self.port_stats.invalidate() + + ################# stream printout ###################### + def generate_loaded_streams_sum(self, stream_id_list): + if self.state == self.STATE_DOWN or self.state == self.STATE_STREAMS: + return {} + streams_data = {} + + if not stream_id_list: + # if no mask has been provided, apply to all streams on port + stream_id_list = self.streams.keys() + + + streams_data = {stream_id: self.streams[stream_id].metadata.get('stream_sum', ["N/A"] * 6) + for stream_id in stream_id_list + if stream_id in self.streams} + + + return {"referring_file" : "", + "streams" : streams_data} + + @staticmethod + def _generate_stream_metadata(stream): + meta_dict = {} + # create packet stream description + #pkt_bld_obj = packet_builder.CTRexPktBuilder() + #pkt_bld_obj.load_from_stream_obj(compiled_stream_obj) + # generate stream summary based on that + + #next_stream = "None" if stream['next_stream_id']==-1 else stream['next_stream_id'] + + meta_dict['stream_sum'] = OrderedDict([("id", stream.get_id()), + ("packet_type", "FIXME!!!"), + ("length", "FIXME!!!"), + ("mode", "FIXME!!!"), + ("rate_pps", "FIXME!!!"), + ("next_stream", "FIXME!!!") + ]) + return meta_dict + + ################# events handler ###################### + def async_event_port_stopped (self): + self.state = self.STATE_STREAMS + + + def async_event_port_started (self): + self.state = self.STATE_TX + + + def async_event_port_paused (self): + self.state = self.STATE_PAUSE + + + def async_event_port_resumed (self): + self.state = self.STATE_TX + + def async_event_forced_acquired (self): + self.handler = None + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_sim.py new file mode 100644 index 00000000..d61e04bf --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_sim.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Itay Marom +Cisco Systems, Inc. + +Copyright (c) 2015-2015 Cisco Systems, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from trex_stl_exceptions import * +from yaml import YAMLError +from trex_stl_streams import * +from client_utils import parsing_opts +from trex_stl_client import STLClient + +import re +import json + + + +import argparse +import tempfile +import subprocess +import os +from dpkt import pcap +from operator import itemgetter + +class BpSimException(Exception): + pass + +def merge_cap_files (pcap_file_list, out_filename, delete_src = False): + + out_pkts = [] + if not all([os.path.exists(f) for f in pcap_file_list]): + print "failed to merge cap file list...\nnot all files exist\n" + return + + # read all packets to a list + for src in pcap_file_list: + f = open(src, 'r') + reader = pcap.Reader(f) + pkts = reader.readpkts() + out_pkts += pkts + f.close() + if delete_src: + os.unlink(src) + + # sort by the timestamp + out_pkts = sorted(out_pkts, key=itemgetter(0)) + + + out = open(out_filename, 'w') + out_writer = pcap.Writer(out) + + for ts, pkt in out_pkts: + out_writer.writepkt(pkt, ts) + + out.close() + + + +# stateless simulation +class STLSim(object): + def __init__ (self, bp_sim_path = None, handler = 0, port_id = 0): + + if not bp_sim_path: + # auto find scripts + m = re.match(".*/trex-core", os.getcwd()) + if not m: + raise STLError('cannot find BP sim path, please provide it') + + self.bp_sim_path = os.path.join(m.group(0), 'scripts') + + else: + self.bp_sim_path = bp_sim_path + + # dummies + self.handler = handler + self.port_id = port_id + + + def generate_start_cmd (self, mult = "1", force = True, duration = -1): + return {"id":1, + "jsonrpc": "2.0", + "method": "start_traffic", + "params": {"handler": self.handler, + "force": force, + "port_id": self.port_id, + "mul": parsing_opts.decode_multiplier(mult), + "duration": duration} + } + + + + # run command + # input_list - a list of streams or YAML files + # outfile - pcap file to save output, if None its a dry run + # dp_core_count - how many DP cores to use + # dp_core_index - simulate only specific dp core without merging + # is_debug - debug or release image + # pkt_limit - how many packets to simulate + # mult - multiplier + # mode - can be 'valgrind, 'gdb', 'json' or 'none' + def run (self, + input_list, + outfile = None, + dp_core_count = 1, + dp_core_index = None, + is_debug = True, + pkt_limit = 5000, + mult = "1", + duration = -1, + mode = 'none'): + + if not mode in ['none', 'gdb', 'valgrind', 'json']: + raise STLArgumentError('mode', mode) + + # listify + input_list = input_list if isinstance(input_list, list) else [input_list] + + # check streams arguments + if not all([isinstance(i, (STLStream, str)) for i in input_list]): + raise STLArgumentError('input_list', input_list) + + # split to two type + input_files = [x for x in input_list if isinstance(x, str)] + stream_list = [x for x in input_list if isinstance(x, STLStream)] + + # handle YAMLs + for input_file in input_files: + stream_list += STLClient.load_profile(input_file) + + + # load streams + cmds_json = [] + for stream in stream_list: + cmd = {"id":1, + "jsonrpc": "2.0", + "method": "add_stream", + "params": {"handler": self.handler, + "port_id": self.port_id, + "stream_id": stream.get_id(), + "stream": stream.to_json()} + } + + cmds_json.append(cmd) + + # generate start command + cmds_json.append(self.generate_start_cmd(mult = mult, + force = True, + duration = duration)) + + if mode == 'json': + print json.dumps(cmds_json, indent = 4, separators=(',', ': '), sort_keys = True) + return + + # start simulation + self.outfile = outfile + self.dp_core_count = dp_core_count + self.dp_core_index = dp_core_index + self.is_debug = is_debug + self.pkt_limit = pkt_limit + self.mult = mult + self.duration = duration, + self.mode = mode + + self.__run(cmds_json) + + + # internal run + def __run (self, cmds_json): + + # write to temp file + f = tempfile.NamedTemporaryFile(delete = False) + f.write(json.dumps(cmds_json)) + f.close() + + # launch bp-sim + try: + self.execute_bp_sim(f.name) + finally: + os.unlink(f.name) + + + + def execute_bp_sim (self, json_filename): + if self.is_debug: + exe = os.path.join(self.bp_sim_path, 'bp-sim-64-debug') + else: + exe = os.path.join(self.bp_sim_path, 'bp-sim-64') + + if not os.path.exists(exe): + raise STLError("'{0}' does not exists, please build it before calling the simulation".format(exe)) + + + cmd = [exe, + '--pcap', + '--sl', + '--cores', + str(self.dp_core_count), + '--limit', + str(self.pkt_limit), + '-f', + json_filename] + + # out or dry + if not self.outfile: + cmd += ['--dry'] + cmd += ['-o', '/dev/null'] + else: + cmd += ['-o', self.outfile] + + if self.dp_core_index != None: + cmd += ['--core_index', str(self.dp_core_index)] + + if self.mode == 'valgrind': + cmd = ['valgrind', '--leak-check=full', '--error-exitcode=1'] + cmd + + elif self.mode == 'gdb': + cmd = ['/bin/gdb', '--args'] + cmd + + print "executing command: '{0}'".format(" ".join(cmd)) + rc = subprocess.call(cmd) + if rc != 0: + raise STLError('simulation has failed with error code {0}'.format(rc)) + + self.merge_results() + + + def merge_results (self): + if not self.outfile: + return + + if self.dp_core_count == 1: + return + + if self.dp_core_index != None: + return + + + print "Mering cores output to a single pcap file...\n" + inputs = ["{0}-{1}".format(self.outfile, index) for index in xrange(0, self.dp_core_count)] + merge_cap_files(inputs, self.outfile, delete_src = True) + + + + +def is_valid_file(filename): + if not os.path.isfile(filename): + raise argparse.ArgumentTypeError("The file '%s' does not exist" % filename) + + return filename + + +def unsigned_int (x): + x = int(x) + if x < 0: + raise argparse.ArgumentTypeError("argument must be >= 0") + + return x + +def setParserOptions(): + parser = argparse.ArgumentParser(prog="stl_sim.py") + + parser.add_argument("-f", + dest ="input_file", + help = "input file in YAML or Python format", + type = is_valid_file, + required=True) + + parser.add_argument("-o", + dest = "output_file", + default = None, + help = "output file in ERF format") + + + parser.add_argument("-c", "--cores", + help = "DP core count [default is 1]", + dest = "dp_core_count", + default = 1, + type = int, + choices = xrange(1, 9)) + + parser.add_argument("-n", "--core_index", + help = "Record only a specific core", + dest = "dp_core_index", + default = None, + type = int) + + parser.add_argument("-r", "--release", + help = "runs on release image instead of debug [default is False]", + action = "store_true", + default = False) + + + parser.add_argument("-l", "--limit", + help = "limit test total packet count [default is 5000]", + default = 5000, + type = unsigned_int) + + parser.add_argument('-m', '--multiplier', + help = parsing_opts.match_multiplier_help, + dest = 'mult', + default = "1", + type = parsing_opts.match_multiplier_strict) + + parser.add_argument('-d', '--duration', + help = "run duration", + dest = 'duration', + default = -1, + type = float) + + + group = parser.add_mutually_exclusive_group() + + group.add_argument("-x", "--valgrind", + help = "run under valgrind [default is False]", + action = "store_true", + default = False) + + group.add_argument("-g", "--gdb", + help = "run under GDB [default is False]", + action = "store_true", + default = False) + + group.add_argument("--json", + help = "generate JSON output only to stdout [default is False]", + action = "store_true", + default = False) + + return parser + + +def validate_args (parser, options): + + if options.dp_core_index: + if not options.dp_core_index in xrange(0, options.dp_core_count): + parser.error("DP core index valid range is 0 to {0}".format(options.dp_core_count - 1)) + + # zero is ok - no limit, but other values must be at least as the number of cores + if (options.limit != 0) and options.limit < options.dp_core_count: + parser.error("limit cannot be lower than number of DP cores") + + +def main (): + parser = setParserOptions() + options = parser.parse_args() + + validate_args(parser, options) + + + + if options.valgrind: + mode = 'valgrind' + elif options.gdb: + mode = 'gdb' + elif options.json: + mode = 'json' + else: + mode = 'none' + + try: + r = STLSim() + r.run(input_list = options.input_file, + outfile = options.output_file, + dp_core_count = options.dp_core_count, + dp_core_index = options.dp_core_index, + is_debug = (not options.release), + pkt_limit = options.limit, + mult = options.mult, + duration = options.duration, + mode = mode) + + except KeyboardInterrupt as e: + print "\n\n*** Caught Ctrl + C... Exiting...\n\n" + exit(1) + + except STLError as e: + print e + exit(1) + + exit(0) + +if __name__ == '__main__': + main() + + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_stats.py new file mode 100644 index 00000000..f880a914 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_stats.py @@ -0,0 +1,581 @@ +#!/router/bin/python + +from client_utils import text_tables +from common.text_opts import format_text, format_threshold, format_num + +from trex_stl_async_client import CTRexAsyncStats + +from collections import namedtuple, OrderedDict, deque +import copy +import datetime +import time +import re +import math +import copy + +GLOBAL_STATS = 'g' +PORT_STATS = 'p' +PORT_STATUS = 'ps' +ALL_STATS_OPTS = {GLOBAL_STATS, PORT_STATS, PORT_STATUS} +COMPACT = {GLOBAL_STATS, PORT_STATS} + +ExportableStats = namedtuple('ExportableStats', ['raw_data', 'text_table']) + +# use to calculate diffs relative to the previous values +# for example, BW +def calculate_diff (samples): + total = 0.0 + + weight_step = 1.0 / sum(xrange(0, len(samples))) + weight = weight_step + + for i in xrange(0, len(samples) - 1): + current = samples[i] if samples[i] > 0 else 1 + next = samples[i + 1] if samples[i + 1] > 0 else 1 + + s = 100 * ((float(next) / current) - 1.0) + + # block change by 100% + total += (min(s, 100) * weight) + weight += weight_step + + return total + + +# calculate by absolute values and not relatives (useful for CPU usage in % and etc.) +def calculate_diff_raw (samples): + total = 0.0 + + weight_step = 1.0 / sum(xrange(0, len(samples))) + weight = weight_step + + for i in xrange(0, len(samples) - 1): + current = samples[i] + next = samples[i + 1] + + total += ( (next - current) * weight ) + weight += weight_step + + return total + + +class CTRexInfoGenerator(object): + """ + This object is responsible of generating stats and information from objects maintained at + STLClient and the ports. + """ + + def __init__(self, global_stats_ref, ports_dict_ref): + self._global_stats = global_stats_ref + self._ports_dict = ports_dict_ref + + def generate_single_statistic(self, port_id_list, statistic_type): + if statistic_type == GLOBAL_STATS: + return self._generate_global_stats() + elif statistic_type == PORT_STATS: + return self._generate_port_stats(port_id_list) + pass + elif statistic_type == PORT_STATUS: + return self._generate_port_status(port_id_list) + else: + # ignore by returning empty object + return {} + + def generate_streams_info(self, port_id_list, stream_id_list): + relevant_ports = self.__get_relevant_ports(port_id_list) + + return_data = OrderedDict() + + for port_obj in relevant_ports: + streams_data = self._generate_single_port_streams_info(port_obj, stream_id_list) + if not streams_data: + continue + hdr_key = "Port {port}: {yaml_file}".format(port= port_obj.port_id, + yaml_file= streams_data.raw_data.get('referring_file', '')) + + # TODO: test for other ports with same stream structure, and join them + return_data[hdr_key] = streams_data + + return return_data + + def _generate_global_stats(self): + stats_data = self._global_stats.generate_stats() + + # build table representation + stats_table = text_tables.TRexTextInfo() + stats_table.set_cols_align(["l", "l"]) + + stats_table.add_rows([[k.replace("_", " ").title(), v] + for k, v in stats_data.iteritems()], + header=False) + + return {"global_statistics": ExportableStats(stats_data, stats_table)} + + def _generate_port_stats(self, port_id_list): + relevant_ports = self.__get_relevant_ports(port_id_list) + + return_stats_data = {} + per_field_stats = OrderedDict([("owner", []), + ("state", []), + ("--", []), + ("Tx bps", []), + ("Tx pps", []), + + ("---", []), + ("Rx bps", []), + ("Rx pps", []), + + ("----", []), + ("opackets", []), + ("ipackets", []), + ("obytes", []), + ("ibytes", []), + ("tx-bytes", []), + ("rx-bytes", []), + ("tx-pkts", []), + ("rx-pkts", []), + + ("-----", []), + ("oerrors", []), + ("ierrors", []), + + ] + ) + + total_stats = CPortStats(None) + + for port_obj in relevant_ports: + # fetch port data + port_stats = port_obj.generate_port_stats() + + total_stats += port_obj.port_stats + + # populate to data structures + return_stats_data[port_obj.port_id] = port_stats + self.__update_per_field_dict(port_stats, per_field_stats) + + total_cols = len(relevant_ports) + header = ["port"] + [port.port_id for port in relevant_ports] + + if (total_cols > 1): + self.__update_per_field_dict(total_stats.generate_stats(), per_field_stats) + header += ['total'] + total_cols += 1 + + stats_table = text_tables.TRexTextTable() + stats_table.set_cols_align(["l"] + ["r"] * total_cols) + stats_table.set_cols_width([10] + [17] * total_cols) + stats_table.set_cols_dtype(['t'] + ['t'] * total_cols) + + stats_table.add_rows([[k] + v + for k, v in per_field_stats.iteritems()], + header=False) + + stats_table.header(header) + + return {"port_statistics": ExportableStats(return_stats_data, stats_table)} + + def _generate_port_status(self, port_id_list): + relevant_ports = self.__get_relevant_ports(port_id_list) + + return_stats_data = {} + per_field_status = OrderedDict([("type", []), + ("maximum", []), + ("status", []) + ] + ) + + for port_obj in relevant_ports: + # fetch port data + # port_stats = self._async_stats.get_port_stats(port_obj.port_id) + port_status = port_obj.generate_port_status() + + # populate to data structures + return_stats_data[port_obj.port_id] = port_status + + self.__update_per_field_dict(port_status, per_field_status) + + stats_table = text_tables.TRexTextTable() + stats_table.set_cols_align(["l"] + ["c"]*len(relevant_ports)) + stats_table.set_cols_width([10] + [20] * len(relevant_ports)) + + stats_table.add_rows([[k] + v + for k, v in per_field_status.iteritems()], + header=False) + stats_table.header(["port"] + [port.port_id + for port in relevant_ports]) + + return {"port_status": ExportableStats(return_stats_data, stats_table)} + + def _generate_single_port_streams_info(self, port_obj, stream_id_list): + + return_streams_data = port_obj.generate_loaded_streams_sum(stream_id_list) + + if not return_streams_data.get("streams"): + # we got no streams available + return None + + # FORMAT VALUES ON DEMAND + + # because we mutate this - deep copy before + return_streams_data = copy.deepcopy(return_streams_data) + + for stream_id, stream_id_sum in return_streams_data['streams'].iteritems(): + stream_id_sum['rate_pps'] = format_num(stream_id_sum['rate_pps'], suffix='pps') + stream_id_sum['packet_type'] = self._trim_packet_headers(stream_id_sum['packet_type'], 20) + + info_table = text_tables.TRexTextTable() + info_table.set_cols_align(["c"] + ["l"] + ["r"] + ["c"] + ["r"] + ["c"]) + info_table.set_cols_width([10] + [20] + [8] + [16] + [10] + [12]) + info_table.set_cols_dtype(["t"] + ["t"] + ["t"] + ["t"] + ["t"] + ["t"]) + + info_table.add_rows([v.values() + for k, v in return_streams_data['streams'].iteritems()], + header=False) + info_table.header(["ID", "packet type", "length", "mode", "rate", "next stream"]) + + return ExportableStats(return_streams_data, info_table) + + + def __get_relevant_ports(self, port_id_list): + # fetch owned ports + ports = [port_obj + for _, port_obj in self._ports_dict.iteritems() + if port_obj.port_id in port_id_list] + + # display only the first FOUR options, by design + if len(ports) > 4: + print format_text("[WARNING]: ", 'magenta', 'bold'), format_text("displaying up to 4 ports", 'magenta') + ports = ports[:4] + return ports + + def __update_per_field_dict(self, dict_src_data, dict_dest_ref): + for key, val in dict_src_data.iteritems(): + if key in dict_dest_ref: + dict_dest_ref[key].append(val) + + @staticmethod + def _trim_packet_headers(headers_str, trim_limit): + if len(headers_str) < trim_limit: + # do nothing + return headers_str + else: + return (headers_str[:trim_limit-3] + "...") + + + +class CTRexStats(object): + """ This is an abstract class to represent a stats object """ + + def __init__(self): + self.reference_stats = {} + self.latest_stats = {} + self.last_update_ts = time.time() + self.history = deque(maxlen = 10) + + def __getitem__(self, item): + # override this to allow quick and clean access to fields + if not item in self.latest_stats: + return "N/A" + + # item must exist + m = re.search('_(([a-z])ps)$', item) + if m: + # this is a non-relative item + unit = m.group(2) + if unit == "b": + return self.get(item, format=True, suffix="b/sec") + elif unit == "p": + return self.get(item, format=True, suffix="pkt/sec") + else: + return self.get(item, format=True, suffix=m.group(1)) + + m = re.search('^[i|o](a-z+)$', item) + if m: + # this is a non-relative item + type = m.group(1) + if type == "bytes": + return self.get_rel(item, format=True, suffix="B") + elif type == "packets": + return self.get_rel(item, format=True, suffix="pkts") + else: + # do not format with suffix + return self.get_rel(item, format=True) + + # can't match to any known pattern, return N/A + return "N/A" + + + def generate_stats(self): + # must be implemented by designated classes (such as port/ global stats) + raise NotImplementedError() + + def update(self, snapshot): + # update + self.latest_stats = snapshot + self.history.append(snapshot) + + diff_time = time.time() - self.last_update_ts + + # 3 seconds is too much - this is the new reference + if (not self.reference_stats) or (diff_time > 3): + self.reference_stats = self.latest_stats + + self.last_update_ts = time.time() + + + def clear_stats(self): + self.reference_stats = self.latest_stats + + + def invalidate (self): + self.latest_stats = {} + + def get(self, field, format=False, suffix=""): + if not field in self.latest_stats: + return "N/A" + if not format: + return self.latest_stats[field] + else: + return format_num(self.latest_stats[field], suffix) + + def get_rel(self, field, format=False, suffix=""): + if not field in self.latest_stats: + return "N/A" + + if not format: + if not field in self.reference_stats: + print "REF: " + str(self.reference_stats) + print "BASE: " + str(self.latest_stats) + + return (self.latest_stats[field] - self.reference_stats[field]) + else: + return format_num(self.latest_stats[field] - self.reference_stats[field], suffix) + + # get trend for a field + def get_trend (self, field, use_raw = False, percision = 10.0): + if not field in self.latest_stats: + return 0 + + # not enough history - no trend + if len(self.history) < 5: + return 0 + + # absolute value is too low 0 considered noise + if self.latest_stats[field] < percision: + return 0 + + field_samples = [sample[field] for sample in self.history] + + if use_raw: + return calculate_diff_raw(field_samples) + else: + return calculate_diff(field_samples) + + + def get_trend_gui (self, field, show_value = False, use_raw = False, up_color = 'red', down_color = 'green'): + v = self.get_trend(field, use_raw) + + value = abs(v) + arrow = u'\u25b2' if v > 0 else u'\u25bc' + color = up_color if v > 0 else down_color + + # change in 1% is not meaningful + if value < 1: + return "" + + elif value > 5: + + if show_value: + return format_text(u"{0}{0}{0} {1:.2f}%".format(arrow,v), color) + else: + return format_text(u"{0}{0}{0}".format(arrow), color) + + elif value > 2: + + if show_value: + return format_text(u"{0}{0} {1:.2f}%".format(arrow,v), color) + else: + return format_text(u"{0}{0}".format(arrow), color) + + else: + if show_value: + return format_text(u"{0} {1:.2f}%".format(arrow,v), color) + else: + return format_text(u"{0}".format(arrow), color) + + + +class CGlobalStats(CTRexStats): + + def __init__(self, connection_info, server_version, ports_dict_ref): + super(CGlobalStats, self).__init__() + self.connection_info = connection_info + self.server_version = server_version + self._ports_dict = ports_dict_ref + + def get_stats (self): + stats = {} + + # absolute + stats['cpu_util'] = self.get("m_cpu_util") + stats['tx_bps'] = self.get("m_tx_bps") + stats['tx_pps'] = self.get("m_tx_pps") + + stats['rx_bps'] = self.get("m_rx_bps") + stats['rx_pps'] = self.get("m_rx_pps") + stats['rx_drop_bps'] = self.get("m_rx_drop_bps") + + # relatives + stats['queue_full'] = self.get_rel("m_total_queue_full") + + return stats + + + def generate_stats(self): + return OrderedDict([("connection", "{host}, Port {port}".format(host=self.connection_info.get("server"), + port=self.connection_info.get("sync_port"))), + ("version", "{ver}, UUID: {uuid}".format(ver=self.server_version.get("version", "N/A"), + uuid="N/A")), + + ("cpu_util", u"{0}% {1}".format( format_threshold(self.get("m_cpu_util"), [85, 100], [0, 85]), + self.get_trend_gui("m_cpu_util", use_raw = True))), + + (" ", ""), + + ("total_tx", u"{0} {1}".format( self.get("m_tx_bps", format=True, suffix="b/sec"), + self.get_trend_gui("m_tx_bps"))), + + ("total_rx", u"{0} {1}".format( self.get("m_rx_bps", format=True, suffix="b/sec"), + self.get_trend_gui("m_rx_bps"))), + + ("total_pps", u"{0} {1}".format( self.get("m_tx_pps", format=True, suffix="pkt/sec"), + self.get_trend_gui("m_tx_pps"))), + + (" ", ""), + + ("drop_rate", "{0}".format( format_num(self.get("m_rx_drop_bps"), + suffix = 'b/sec', + opts = 'green' if (self.get("m_rx_drop_bps")== 0) else 'red'))), + + ("queue_full", "{0}".format( format_num(self.get_rel("m_total_queue_full"), + suffix = 'pkts', + compact = False, + opts = 'green' if (self.get_rel("m_total_queue_full")== 0) else 'red'))), + + ] + ) + +class CPortStats(CTRexStats): + + def __init__(self, port_obj): + super(CPortStats, self).__init__() + self._port_obj = port_obj + + @staticmethod + def __merge_dicts (target, src): + for k, v in src.iteritems(): + if k in target: + target[k] += v + else: + target[k] = v + + + def __add__ (self, x): + if not isinstance(x, CPortStats): + raise TypeError("cannot add non stats object to stats") + + # main stats + if not self.latest_stats: + self.latest_stats = {} + + self.__merge_dicts(self.latest_stats, x.latest_stats) + + # reference stats + if x.reference_stats: + if not self.reference_stats: + self.reference_stats = x.reference_stats.copy() + else: + self.__merge_dicts(self.reference_stats, x.reference_stats) + + # history + if not self.history: + self.history = copy.deepcopy(x.history) + else: + for h1, h2 in zip(self.history, x.history): + self.__merge_dicts(h1, h2) + + return self + + # for port we need to do something smarter + def get_stats (self): + stats = {} + + stats['opackets'] = self.get_rel("opackets") + stats['ipackets'] = self.get_rel("ipackets") + stats['obytes'] = self.get_rel("obytes") + stats['ibytes'] = self.get_rel("ibytes") + stats['oerrors'] = self.get_rel("oerrors") + stats['ierrors'] = self.get_rel("ierrors") + stats['tx_bps'] = self.get("m_total_tx_bps") + stats['tx_pps'] = self.get("m_total_tx_pps") + stats['rx_bps'] = self.get("m_total_rx_bps") + stats['rx_pps'] = self.get("m_total_rx_pps") + + return stats + + + def generate_stats(self): + + state = self._port_obj.get_port_state_name() if self._port_obj else "" + if state == "ACTIVE": + state = format_text(state, 'green', 'bold') + elif state == "PAUSE": + state = format_text(state, 'magenta', 'bold') + else: + state = format_text(state, 'bold') + + return {"owner": self._port_obj.user if self._port_obj else "", + "state": "{0}".format(state), + + "--": " ", + "---": " ", + "----": " ", + "-----": " ", + + "Tx bps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_bps", show_value = False), + self.get("m_total_tx_bps", format = True, suffix = "bps")), + + "Rx bps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_bps", show_value = False), + self.get("m_total_rx_bps", format = True, suffix = "bps")), + + "Tx pps": u"{0} {1}".format(self.get_trend_gui("m_total_tx_pps", show_value = False), + self.get("m_total_tx_pps", format = True, suffix = "pps")), + + "Rx pps": u"{0} {1}".format(self.get_trend_gui("m_total_rx_pps", show_value = False), + self.get("m_total_rx_pps", format = True, suffix = "pps")), + + "opackets" : self.get_rel("opackets"), + "ipackets" : self.get_rel("ipackets"), + "obytes" : self.get_rel("obytes"), + "ibytes" : self.get_rel("ibytes"), + + "tx-bytes": self.get_rel("obytes", format = True, suffix = "B"), + "rx-bytes": self.get_rel("ibytes", format = True, suffix = "B"), + "tx-pkts": self.get_rel("opackets", format = True, suffix = "pkts"), + "rx-pkts": self.get_rel("ipackets", format = True, suffix = "pkts"), + + "oerrors" : format_num(self.get_rel("oerrors"), + compact = False, + opts = 'green' if (self.get_rel("oerrors")== 0) else 'red'), + + "ierrors" : format_num(self.get_rel("ierrors"), + compact = False, + opts = 'green' if (self.get_rel("ierrors")== 0) else 'red'), + + } + + + +if __name__ == "__main__": + pass diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_std.py b/scripts/automation/trex_control_plane/stl/trex_stl_std.py new file mode 100644 index 00000000..72a5ea52 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_std.py @@ -0,0 +1,67 @@ +from trex_stl_streams import * +from trex_stl_packet_builder_scapy import * + +# map ports +# will destroy all streams/data on the ports +def stl_map_ports (client, ports = None): + + # by default use all ports + if ports == None: + ports = client.get_all_ports() + + # reset the ports + client.reset(ports) + + # generate streams + base_pkt = CScapyTRexPktBuilder(pkt = Ether()/IP()) + + pkts = 1 + for port in ports: + stream = STLStream(packet = base_pkt, + mode = STLTXSingleBurst(pps = 100000, total_pkts = pkts)) + + client.add_streams(stream, [port]) + pkts = pkts * 2 + + # inject + client.clear_stats() + client.start(ports, mult = "1mpps") + client.wait_on_traffic(ports) + + stats = client.get_stats() + + # cleanup + client.reset(ports = ports) + + table = {} + for port in ports: + table[port] = None + + for port in ports: + ipackets = stats[port]["ipackets"] + + exp = 1 + while ipackets >= exp: + if ((ipackets & exp) == (exp)): + source = int(math.log(exp, 2)) + table[source] = port + + exp *= 2 + + if not all(x != None for x in table.values()): + raise STLError('unable to map ports') + + dir_a = set() + dir_b = set() + for src, dst in table.iteritems(): + # src is not in + if src not in (dir_a, dir_b): + if dst in dir_a: + dir_b.add(src) + else: + dir_a.add(src) + + table['dir'] = [list(dir_a), list(dir_b)] + + return table + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_streams.py new file mode 100644 index 00000000..d5cba9e2 --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_streams.py @@ -0,0 +1,230 @@ +#!/router/bin/python + +from trex_stl_exceptions import * +from trex_stl_packet_builder_interface import CTrexPktBuilderInterface +from trex_stl_packet_builder_scapy import CScapyTRexPktBuilder, Ether, IP +from collections import OrderedDict, namedtuple + +from trex_control_plane.client_utils.yaml_utils import * + +from dpkt import pcap +import random +import yaml +import base64 + +# base class for TX mode +class STLTXMode(object): + def __init__ (self): + self.fields = {} + + def to_json (self): + return self.fields + + +# continuous mode +class STLTXCont(STLTXMode): + + def __init__ (self, pps = 1): + + if not isinstance(pps, (int, float)): + raise STLArgumentError('pps', pps) + + super(STLTXCont, self).__init__() + + self.fields['type'] = 'continuous' + self.fields['pps'] = pps + + +# single burst mode +class STLTXSingleBurst(STLTXMode): + + def __init__ (self, pps = 1, total_pkts = 1): + + if not isinstance(pps, (int, float)): + raise STLArgumentError('pps', pps) + + if not isinstance(total_pkts, int): + raise STLArgumentError('total_pkts', total_pkts) + + super(STLTXSingleBurst, self).__init__() + + self.fields['type'] = 'single_burst' + self.fields['pps'] = pps + self.fields['total_pkts'] = total_pkts + + +# multi burst mode +class STLTXMultiBurst(STLTXMode): + + def __init__ (self, + pps = 1, + pkts_per_burst = 1, + ibg = 0.0, + count = 1): + + if not isinstance(pps, (int, float)): + raise STLArgumentError('pps', pps) + + if not isinstance(pkts_per_burst, int): + raise STLArgumentError('pkts_per_burst', pkts_per_burst) + + if not isinstance(ibg, (int, float)): + raise STLArgumentError('ibg', ibg) + + if not isinstance(count, int): + raise STLArgumentError('count', count) + + super(STLTXMultiBurst, self).__init__() + + self.fields['type'] = 'multi_burst' + self.fields['pps'] = pps + self.fields['pkts_per_burst'] = pkts_per_burst + self.fields['ibg'] = ibg + self.fields['count'] = count + + +class STLStream(object): + + def __init__ (self, + packet = None, + mode = STLTXCont(1), + enabled = True, + self_start = True, + isg = 0.0, + rx_stats = None, + next_stream_id = -1, + stream_id = None): + + # type checking + if not isinstance(mode, STLTXMode): + raise STLArgumentError('mode', mode) + + if packet and not isinstance(packet, CTrexPktBuilderInterface): + raise STLArgumentError('packet', packet) + + if not isinstance(enabled, bool): + raise STLArgumentError('enabled', enabled) + + if not isinstance(self_start, bool): + raise STLArgumentError('self_start', self_start) + + if not isinstance(isg, (int, float)): + raise STLArgumentError('isg', isg) + + if (type(mode) == STLTXCont) and (next_stream_id != -1): + raise STLError("continuous stream cannot have a next stream ID") + + self.fields = {} + + # use a random 31 bit for ID + self.fields['stream_id'] = stream_id if stream_id is not None else random.getrandbits(31) + + # basic fields + self.fields['enabled'] = enabled + self.fields['self_start'] = self_start + self.fields['isg'] = isg + + self.fields['next_stream_id'] = next_stream_id + + # mode + self.fields['mode'] = mode.to_json() + + self.fields['packet'] = {} + self.fields['vm'] = {} + + if not packet: + packet = CScapyTRexPktBuilder(pkt = Ether()/IP()) + + # packet builder + packet.compile() + # packet and VM + self.fields['packet'] = packet.dump_pkt() + self.fields['vm'] = packet.get_vm_data() + + self.fields['rx_stats'] = {} + if not rx_stats: + self.fields['rx_stats']['enabled'] = False + + + def __str__ (self): + return json.dumps(self.fields, indent = 4, separators=(',', ': '), sort_keys = True) + + def to_json (self): + return self.fields + + def get_id (self): + return self.fields['stream_id'] + + + + @staticmethod + def dump_to_yaml (stream_list, yaml_file = None): + + # type check + if isinstance(stream_list, STLStream): + stream_list = [stream_list] + + if not all([isinstance(stream, STLStream) for stream in stream_list]): + raise STLArgumentError('stream_list', stream_list) + + + names = {} + for i, stream in enumerate(stream_list): + names[stream.get_id()] = "stream-{0}".format(i) + + yaml_lst = [] + for stream in stream_list: + + fields = dict(stream.fields) + + # handle the next stream id + if fields['next_stream_id'] == -1: + del fields['next_stream_id'] + + else: + if not stream.get_id() in names: + raise STLError('broken dependencies in stream list') + + fields['next_stream'] = names[stream.get_id()] + + # add to list + yaml_lst.append({'name': names[stream.get_id()], 'stream': fields}) + + # write to file + x = yaml.dump(yaml_lst, default_flow_style=False) + if yaml_file: + with open(yaml_file, 'w') as f: + f.write(x) + + return x + + + @staticmethod + def load_from_yaml (yaml_file): + + with open(yaml_file, 'r') as f: + yaml_str = f.read() + + + # load YAML + lst = yaml.load(yaml_str) + + # decode to streams + streams = [] + for stream in lst: + # for defaults + defaults = STLStream() + s = STLStream(packet = None, + mode = STLTXCont(1), + enabled = True, + self_start = True, + isg = 0.0, + rx_stats = None, + next_stream_id = -1, + stream_id = None + ) + + streams.append(s) + + return streams + diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_types.py b/scripts/automation/trex_control_plane/stl/trex_stl_types.py new file mode 100644 index 00000000..a7ddacea --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_types.py @@ -0,0 +1,95 @@ + +from collections import namedtuple +from common.text_opts import * + +RpcCmdData = namedtuple('RpcCmdData', ['method', 'params']) + +class RpcResponseStatus(namedtuple('RpcResponseStatus', ['success', 'id', 'msg'])): + __slots__ = () + def __str__(self): + return "{id:^3} - {msg} ({stat})".format(id=self.id, + msg=self.msg, + stat="success" if self.success else "fail") + +# simple class to represent complex return value +class RC(): + + def __init__ (self, rc = None, data = None, is_warn = False): + self.rc_list = [] + + if (rc != None): + tuple_rc = namedtuple('RC', ['rc', 'data', 'is_warn']) + self.rc_list.append(tuple_rc(rc, data, is_warn)) + + def __nonzero__ (self): + return self.good() + + + def add (self, rc): + self.rc_list += rc.rc_list + + def good (self): + return all([x.rc for x in self.rc_list]) + + def bad (self): + return not self.good() + + def warn (self): + return any([x.is_warn for x in self.rc_list]) + + def data (self): + d = [x.data if x.rc else "" for x in self.rc_list] + return (d if len(d) != 1 else d[0]) + + def err (self): + e = [x.data if not x.rc else "" for x in self.rc_list] + return (e if len(e) != 1 else e[0]) + + def __str__ (self): + s = "" + for x in self.rc_list: + if x.data: + s += format_text("\n{0}".format(x.data), 'bold') + return s + + def prn_func (self, msg, newline = True): + if newline: + print msg + else: + print msg, + + def annotate (self, log_func = None, desc = None, show_status = True): + + if not log_func: + log_func = self.prn_func + + if desc: + log_func(format_text('\n{:<60}'.format(desc), 'bold'), newline = False) + else: + log_func("") + + if self.bad(): + # print all the errors + print "" + for x in self.rc_list: + if not x.rc: + log_func(format_text("\n{0}".format(x.data), 'bold')) + + print "" + if show_status: + log_func(format_text("[FAILED]\n", 'red', 'bold')) + + + else: + if show_status: + log_func(format_text("[SUCCESS]\n", 'green', 'bold')) + + +def RC_OK(data = ""): + return RC(True, data) + +def RC_ERR (err): + return RC(False, err) + +def RC_WARN (warn): + return RC(True, warn, is_warn = True) -- cgit 1.2.3-korg