import os
from .stl_general_test import CStlGeneral_Test, CTRexScenario
from trex_stl_lib.api import *
import pprint
def avg (values):
return (sum(values) / float(len(values)))
# performance report object
class PerformanceReport(object):
GOLDEN_NORMAL = 1
GOLDEN_FAIL = 2
GOLDEN_BETTER = 3
def __init__ (self,
scenario,
machine_name,
core_count,
avg_cpu,
avg_gbps,
avg_mpps,
avg_gbps_per_core,
avg_mpps_per_core,
):
self.scenario = scenario
self.machine_name = machine_name
self.core_count = core_count
self.avg_cpu = avg_cpu
self.avg_gbps = avg_gbps
self.avg_mpps = avg_mpps
self.avg_gbps_per_core = avg_gbps_per_core
self.avg_mpps_per_core = avg_mpps_per_core
def show (self):
print("\n")
print("scenario: {0}".format(self.scenario))
print("machine name: {0}".format(self.machine_name))
print("DP core count: {0}".format(self.core_count))
print("average CPU: {0}".format(self.avg_cpu))
print("average Gbps: {0}".format(self.avg_gbps))
print("average Mpps: {0}".format(self.avg_mpps))
print("average pkt size (bytes): {0}".format( (self.avg_gbps * 1000 / 8) / self.avg_mpps))
print("average Gbps per core (at 100% CPU): {0}".format(self.avg_gbps_per_core))
print("average Mpps per core (at 100% CPU): {0}".format(self.avg_mpps_per_core))
def check_golden (self, golden_mpps):
if self.avg_mpps_per_core < golden_mpps['min']:
return self.GOLDEN_FAIL
if self.avg_mpps_per_core > golden_mpps['max']:
return self.GOLDEN_BETTER
return self.GOLDEN_NORMAL
def report_to_analytics(self, ga, golden_mpps):
print("\n* Reporting to GA *\n")
ga.gaAddTestQuery(TestName = self.scenario,
TRexMode = 'stl',
SetupName = self.machine_name,
TestType = 'performance',
Mppspc = self.avg_mpps_per_core,
ActionNumber = os.getenv("BUILD_ID","n/a"),
GoldenMin = golden_mpps['min'],
GoldenMax = golden_mpps['max'])
ga.emptyAndReportQ()
def norm_senario (self):
s=self.scenario
s='+'.join(s.split(' '));
s='+'.join(s.split('-'));
s='+'.join(s.split(','));
l=s.split('+')
lr=[]
for obj in l:
if len(obj):
lr.append(obj);
s='-'.join(lr);
return(s);
def report_to_elk(self, elk,elk_obj, golden_mpps):
print("\n* Reporting to elk *\n")
elk_obj['test']={ "name" : self.norm_senario(),
"type" : "stateless",
"cores" : self.core_count,
"cpu%" : self.avg_cpu,
"mpps" : self.avg_mpps,
"streams_count" : 1,
"mpps_pc" : self.avg_mpps_per_core,
"gbps_pc" : self.avg_gbps_per_core,
"gbps" : self.avg_gbps,
"avg-pktsize" : ((1000.0*self.avg_gbps/(8.0*self.avg_mpps))),
"latecny" : { "min" : -1.0,
"max" : -1.0,
"avr" : -1.0
}
};
#pprint.pprint(elk_obj);
# push to elk
elk.perf.push_data(elk_obj)
class STLPerformance_Test(CStlGeneral_Test):
"""Tests for stateless client"""
def setUp(self):
CStlGeneral_Test.setUp(self)
self.c = CTRexScenario.stl_trex
self.c.connect()
self.c.reset()
def tearDown (self):
CStlGeneral_Test.tearDown(self)
def build_perf_profile_vm (self, pkt_size, cache_size = None):
size = pkt_size - 4; # HW will add 4 bytes ethernet FCS
src_ip = '16.0.0.1'
dst_ip = '48.0.0.1'
base_pkt = Ether()/IP(src=src_ip,dst=dst_ip)/UDP(dport=12,sport=1025)
pad = max(0, size - len(base_pkt)) * 'x'
vm = STLScVmRaw( [ STLVmFlowVar ( "ip_src", min_value="10.0.0.1", max_value="10.0.0.255", size=4, step=1,op="inc"),
STLVmWrFlowVar (fv_name="ip_src", pkt_offset= "IP.src" ),
STLVmFixIpv4(offset = "IP")
],
cache_size = cache_size
);
pkt = STLPktBuilder(pkt = base_pkt/pad, vm = vm)
return STLStream(packet = pkt, mode = STLTXCont())
def build_perf_profile_syn_attack (self, pkt_size):
size = pkt_size - 4; # HW will add 4 bytes ethernet FCS
# TCP SYN
base_pkt = Ether()/IP(dst="48.0.0.1")/TCP(dport=80,flags="S")
pad = max(0, size - len(base_pkt)) * 'x'
# vm
vm = STLScVmRaw( [ STLVmFlowVar(name="ip_src",
min_value="16.0.0.0",
max_value="18.0.0.254",
size=4, op="random"),
STLVmFlowVar(name="src_port",
min_value=1025,
max_value=65000,
size=2, op="random"),
STLVmWrFlowVar(fv_name="ip_src", pkt_offset= "IP.src" ),
STLVmFixIpv4(offset = "IP"), # fix checksum
STLVmWrFlowVar(fv_name="src_port",
pkt_offset= "TCP.sport") # fix udp len
]
)
pkt = STLPktBuilder(pkt = base_pkt,
vm = vm)
return STLStream(packet = pkt,
random_seed = 0x1234,# can be remove. will give the same random value any run
mode = STLTXCont())
# single CPU, VM, no cache, 64 bytes
def test_performance_vm_single_cpu (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "VM - 64 bytes, single CPU"
scenario_cfg['streams'] = self.build_perf_profile_vm(64)
scenario_cfg['core_count'] = 1
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
# single CPU, VM, cached, 64 bytes
def test_performance_vm_single_cpu_cached (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "VM - 64 bytes, single CPU, cache size 1024"
scenario_cfg['streams'] = self.build_perf_profile_vm(64, cache_size = 1024)
scenario_cfg['core_count'] = 1
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
# single CPU, syn attack, 64 bytes
def test_performance_syn_attack_single_cpu (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "syn attack - 64 bytes, single CPU"
scenario_cfg['streams'] = self.build_perf_profile_syn_attack(64)
scenario_cfg['core_count'] = 1
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
# two CPUs, VM, no cache, 64 bytes
def test_performance_vm_multi_cpus (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "VM - 64 bytes, multi CPUs"
scenario_cfg['streams'] = self.build_perf_profile_vm(64)
scenario_cfg['core_count'] = setup_cfg['core_count']
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
# multi CPUs, VM, cached, 64 bytes
def test_performance_vm_multi_cpus_cached (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "VM - 64 bytes, multi CPU, cache size 1024"
scenario_cfg['streams'] = self.build_perf_profile_vm(64, cache_size = 1024)
scenario_cfg['core_count'] = setup_cfg['core_count']
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
# multi CPUs, syn attack, 64 bytes
def test_performance_syn_attack_multi_cpus (self):
setup_cfg = self.get_benchmark_param('cfg')
scenario_cfg = {}
scenario_cfg['name'] = "syn attack - 64 bytes, multi CPUs"
scenario_cfg['streams'] = self.build_perf_profile_syn_attack(64)
scenario_cfg['core_count'] = setup_cfg['core_count']
scenario_cfg['mult'] = setup_cfg['mult']
scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
self.execute_single_scenario(scenario_cfg)
############################################# test's infra functions ###########################################
def execute_single_scenario (self, scenario_cfg):
golden = scenario_cfg['mpps_per_core_golden']
report = self.execute_single_scenario_iteration(scenario_cfg)
if self.GAManager:
report.report_to_analytics(self.GAManager, golden)
#report to elk
if self.elk:
elk_obj = self.get_elk_obj()
report.report_to_elk(self.elk,elk_obj, golden)
rc = report.check_golden(golden)
if rc == PerformanceReport.GOLDEN_NORMAL or rc == PerformanceReport.GOLDEN_BETTER:
return
print("\n*** Measured Mpps per core '{0}' is lower than expected golden '{1}'".format(report.avg_mpps_per_core, scenario_cfg['mpps_per_core_golden']))
assert 0, "performance failure"
def execute_single_scenario_iteration (self, scenario_cfg):
print("\nExecuting performance scenario: '{0}'\n".format(scenario_cfg['name']))
self.c.reset(ports = [0])
self.c.add_streams(ports = [0], streams = scenario_cfg['streams'])
# use one core
cores_per_port = self.c.system_info.get('dp_core_count_per_port', 0)
if cores_per_port < scenario_cfg['core_count']:
assert 0, "test configuration requires {0} cores but only {1} per port are available".format(scenario_cfg['core_count'], cores_per_port)
core_mask = (2 ** scenario_cfg['core_count']) - 1
self.c.start(ports = [0], mult = scenario_cfg['mult'], core_mask = [core_mask])
# stablize
print("Step 1 - waiting for stabilization... (10 seconds)")
for _ in range(10):
time.sleep(1)
sys.stdout.write('.')
sys.stdout.flush()
print("\n")
samples = {'cpu' : [], 'bps': [], 'pps': []}
# let the server gather samples
print("Step 2 - Waiting for samples... (60 seconds)")
for i in range(0, 3):
# sample bps/pps
for _ in range(0, 20):
stats = self.c.get_stats(ports = 0)
max_queue_full = 100000 if self.is_VM else 10000
if stats['global'][ 'queue_full'] > max_queue_full:
assert 0, "Queue is full need to tune the multiplier"
# CPU results are not valid cannot use them
samples['bps'].append(stats[0]['tx_bps'])
samples['pps'].append(stats[0]['tx_pps'])
time.sleep(1)
sys.stdout.write('.')
sys.stdout.flush()
# sample CPU per core
rc = self.c._transmit('get_utilization')
if not rc:
raise Exception(rc)
data = rc.data()['cpu']
# filter
data = [s for s in data if s['ports'][0] == 0]
assert len(data) == scenario_cfg['core_count'] , "sampling info does not match core count"
for s in data:
samples['cpu'] += s['history']
stats = self.c.get_stats(ports = 0)
self.c.stop(ports = [0])
avg_values = {k:avg(v) for k, v in samples.items()}
avg_cpu = avg_values['cpu'] * scenario_cfg['core_count']
avg_gbps = avg_values['bps'] / 1e9
avg_mpps = avg_values['pps'] / 1e6
avg_gbps_per_core = avg_gbps * (100.0 / avg_cpu)
avg_mpps_per_core = avg_mpps * (100.0 / avg_cpu)
report = PerformanceReport(scenario = scenario_cfg['name'],
machine_name = CTRexScenario.setup_name,
core_count = scenario_cfg['core_count'],
avg_cpu = avg_cpu,
avg_gbps = avg_gbps,
avg_mpps = avg_mpps,
avg_gbps_per_core = avg_gbps_per_core,
avg_mpps_per_core = avg_mpps_per_core)
report.show()
print("")
golden = scenario_cfg['mpps_per_core_golden']
print("golden Mpps per core (at 100% CPU): min: {0}, max {1}".format(golden['min'], golden['max']))
return report