From e831a539aebac71d8f00c400416f4bef51108610 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Fri, 29 Apr 2016 17:07:14 +0300 Subject: regression: add GA --- .../regression/stateful_tests/trex_general_test.py | 11 +- .../stateless_tests/stl_benchmark_test.py | 7 + scripts/automation/regression/trex.py | 1 + scripts/automation/regression/trex_unit_test.py | 5 + .../stl/trex_stl_lib/utils/GAObjClass.py | 295 +++++++++++++++++++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100755 scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/GAObjClass.py (limited to 'scripts/automation') diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py index e21e2f31..7402175e 100755 --- a/scripts/automation/regression/stateful_tests/trex_general_test.py +++ b/scripts/automation/regression/stateful_tests/trex_general_test.py @@ -58,6 +58,7 @@ class CTRexGeneral_Test(unittest.TestCase): self.trex = CTRexScenario.trex self.trex_crashed = CTRexScenario.trex_crashed self.modes = CTRexScenario.modes + self.GAManager = CTRexScenario.GAManager self.skipping = False self.fail_reasons = [] if not hasattr(self, 'unsupported_modes'): @@ -136,7 +137,7 @@ class CTRexGeneral_Test(unittest.TestCase): if res[name] != float(val): self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val)) - def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = 30, maximal_cpu = 85): + def check_CPU_benchmark (self, trex_res, err = 25, minimal_cpu = 30, maximal_cpu = 85): #cpu_util = float(trex_res.get_last_value("trex-global.data.m_cpu_util")) cpu_util = sum(trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]) / 3.0 # mean of 3 values before last @@ -155,16 +156,22 @@ class CTRexGeneral_Test(unittest.TestCase): # x2 because each thread uses 2 ports and another x2 because each core can use 2 threads test_norm_cpu = 2 * 2 * (100.0 / cpu_util) * trex_tx_bps / (ports_count * cores * 1e6) - print("TRex CPU utilization: %g%%, norm_cpu is : %g Mb/core" % (round(cpu_util), round(test_norm_cpu))) + print("TRex CPU utilization: %g%%, norm_cpu is : %g Mb/core" % (round(cpu_util, 2), round(test_norm_cpu))) expected_norm_cpu = self.get_benchmark_param('bw_per_core') if not expected_norm_cpu: expected_norm_cpu = 1 + calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100) print('Err percent: %s' % calc_error_precent) if calc_error_precent > err: self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu))) + # report benchmarks + if GAManager: + self.GAManager.gaAddAction(Event = 'stateful_test', action = self.get_name(), label = 'bw_per_core', value = int(test_norm_cpu)) + self.GAManager.gaAddAction(Event = 'stateful_test', action = self.get_name(), label = 'bw_per_core_exp', value = int(expected_norm_cpu)) + self.GAManager.emptyAndReportQ() def check_results_gt (self, res, name, val): if res is None: diff --git a/scripts/automation/regression/stateless_tests/stl_benchmark_test.py b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py index 6614b1a6..78abff07 100755 --- a/scripts/automation/regression/stateless_tests/stl_benchmark_test.py +++ b/scripts/automation/regression/stateless_tests/stl_benchmark_test.py @@ -48,6 +48,13 @@ class STLBenchmark_Test(CStlGeneral_Test): print('Done (%ss), CPU util: %4g, bw_per_core: %6sGb/core' % (int(time() - start_time), agv_cpu_util, round(bw_per_core, 2))) # TODO: add check of benchmark based on results from regression + # report benchmarks + if GAManager: + profile_repr = '%s %s' % (self.get_name(), repr(kwargs).replace("'", '')) + self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr, label = 'bw_per_core', value = int(bw_per_core)) + # TODO: report expected once acquired + #self.GAManager.gaAddAction(Event = 'stateless_test', action = profile_repr, label = 'bw_per_core_exp', value = int(expected_norm_cpu)) + self.GAManager.emptyAndReportQ() def tearDown(self): self.stl_trex.reset() diff --git a/scripts/automation/regression/trex.py b/scripts/automation/regression/trex.py index 9459e7c6..7440d76d 100644 --- a/scripts/automation/regression/trex.py +++ b/scripts/automation/regression/trex.py @@ -36,6 +36,7 @@ class CTRexScenario: # logger = None test_types = {'functional_tests': [], 'stateful_tests': [], 'stateless_tests': []} is_copied = False + GAManager = None class CTRexRunner: """This is an instance for generating a CTRexRunner""" diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py index 65126b04..d87cb057 100755 --- a/scripts/automation/regression/trex_unit_test.py +++ b/scripts/automation/regression/trex_unit_test.py @@ -39,6 +39,7 @@ from trex import CTRexScenario from trex_stf_lib.trex_client import * from trex_stf_lib.trex_exceptions import * from trex_stl_lib.api import * +from trex_stl_lib.utils.GAObjClass import GAmanager import trex import socket from pprint import pprint @@ -153,6 +154,9 @@ class CTRexTestConfiguringPlugin(Plugin): parser.add_option('--test-client-package', '--test_client_package', action="store_true", default = False, dest="test_client_package", help="Includes tests of client package.") + parser.add_option('--long', action="store_true", default = False, + dest="long", type = int, + help="Flag of long tests (stability).") def configure(self, options, conf): self.collect_only = options.collect_only @@ -301,6 +305,7 @@ if __name__ == "__main__": xml_name = 'unit_test.xml' if CTRexScenario.setup_dir: CTRexScenario.setup_name = os.path.basename(CTRexScenario.setup_dir) + CTRexScenario.GAManager = GAmanager(GoogleID='UA-75220362-4', UserID=CTRexScenario.setup_name, QueueSize=100, Timeout=5, UserPermission=1, BlockingMode=1, appName='TRex', appVer='1.11.232') #timeout in seconds xml_name = 'report_%s.xml' % CTRexScenario.setup_name xml_arg= '--xunit-file=%s/%s' % (CTRexScenario.report_dir, xml_name) set_report_dir(CTRexScenario.report_dir) diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/GAObjClass.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/GAObjClass.py new file mode 100755 index 00000000..3101b45d --- /dev/null +++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/GAObjClass.py @@ -0,0 +1,295 @@ +#import requests # need external lib for that +try: # Python2 + import Queue + from urllib2 import * +except: # Python3 + import queue as Queue + from urllib.request import * + from urllib.error import * +import threading +import sys +from time import sleep + +""" +GAObjClass is a class destined to send Google Analytics Information. + +cid - unique number per user. +command - the Event Category rubric appears on site. type: TEXT +action - the Event Action rubric appears on site - type: TEXT +label - the Event Label rubric - type: TEXT +value - the event value metric - type: INTEGER + +QUOTAS: +1 single payload - up to 8192Bytes +batched: +A maximum of 20 hits can be specified per request. +The total size of all hit payloads cannot be greater than 16K bytes. +No single hit payload can be greater than 8K bytes. +""" + +url_single = 'http://www.google-analytics.com/collect' #sending single event +url_batched = 'http://www.google-analytics.com/batch' #sending batched events +url_debug = 'http://www.google-analytics.com/debug/collect' #verifying hit is valid +url_conn = 'http://172.217.2.196' # testing internet connection to this address (google-analytics server) + + +#..................................................................class GA_EVENT_ObjClass................................................................ +class GA_EVENT_ObjClass: + def __init__(self,cid,trackerID,command,action,label,value,appName,appVer): + self.cid = cid + self.trackerID = trackerID + self.command = command + self.action = action + self.label = label + self.value = value + self.appName = appName + self.appVer = appVer + self.generate_payload() + self.size = sys.getsizeof(self.payload) + + def generate_payload(self): + self.payload ='v=1&t=event&tid='+str(self.trackerID) + self.payload+='&cid='+str(self.cid) + self.payload+='&ec='+str(self.command) + self.payload+='&ea='+str(self.action) + self.payload+='&el='+str(self.label) + self.payload+='&ev='+str(self.value) + self.payload+='&an='+str(self.appName) + self.payload+='&av='+str(self.appVer) + +#..................................................................class GA_EXCEPTION_ObjClass................................................................ +#ExceptionFatal - BOOLEAN +class GA_EXCEPTION_ObjClass: + def __init__(self,cid,trackerID,ExceptionName,ExceptionFatal,appName,appVer): + self.cid = cid + self.trackerID = trackerID + self.ExceptionName = ExceptionName + self.ExceptionFatal = ExceptionFatal + self.appName = appName + self.appVer = appVer + self.generate_payload() + + def generate_payload(self): + self.payload ='v=1&t=exception&tid='+str(self.trackerID) + self.payload+='&cid='+str(self.cid) + self.payload+='&exd='+str(self.ExceptionName) + self.payload+='&exf='+str(self.ExceptionFatal) + self.payload+='&an='+str(self.appName) + self.payload+='&av='+str(self.appVer) + +#.....................................................................class ga_Thread................................................................. +""" + +Google analytics thread manager: + +will report and empty queue of google analytics items to GA server, every Timeout (parameter given on initialization) +will perform connectivity check every timeout*10 seconds + + +""" + +class ga_Thread (threading.Thread): + def __init__(self,threadID,gManager): + threading.Thread.__init__(self) + self.threadID = threadID + self.gManager = gManager + + def run(self): + keepAliveCounter=0 + #sys.stdout.write('thread started \n') + #sys.stdout.flush() + while True: + if (keepAliveCounter==10): + keepAliveCounter=0 + if (self.gManager.internet_on()==True): + self.gManager.connectedToInternet=1 + else: + self.gManager.connectedToInternet=0 + sleep(self.gManager.Timeout) + keepAliveCounter+=1 + if not self.gManager.GA_q.empty(): + self.gManager.threadLock.acquire(1) +# sys.stdout.write('lock acquired: reporting to GA \n') +# sys.stdout.flush() + if (self.gManager.connectedToInternet==1): + self.gManager.emptyAndReportQ() + self.gManager.threadLock.release() +# sys.stdout.write('finished \n') +# sys.stdout.flush() + + + +#.....................................................................class GAmanager................................................................. +""" + +Google ID - specify tracker property, example: UA-75220362-2 (when the suffix '2' specifies the analytics property profile) + +UserID - unique userID, this will differ between users on GA + +appName - s string to determine app name + +appVer - a string to determine app version + +QueueSize - the size of the queue that holds reported items. once the Queue is full: + on blocking mode: + will block program until next submission to GA server, which will make new space + on non-blocking mode: + will drop new requests + +Timout - the timeout the queue uses between data transmissions. Timeout should be shorter than the time it takes to generate 20 events. MIN VALUE = 11 seconds + +User Permission - the user must accept data transmission, use this flag as 1/0 flag, when UserPermission=1 allows data collection + +BlockingMode - set to 1 if you wish every Google Analytic Object will be submitted and processed, with no drops allowed. + this will block the running of the program until every item is processed + +*** Restriction - Google's restriction for amount of packages being sent per session per second is: 1 event per second, per session. session length is 30min *** +""" +class GAmanager: + def __init__(self,GoogleID,UserID,appName,appVer,QueueSize,Timeout,UserPermission,BlockingMode): + self.UserID = UserID + self.GoogleID = GoogleID + self.QueueSize = QueueSize + self.Timeout = Timeout + self.appName = appName + self.appVer = appVer + self.UserPermission = UserPermission + self.GA_q = Queue.Queue(QueueSize) + self.thread = ga_Thread(UserID,self) + self.threadLock = threading.Lock() + self.BlockingMode = BlockingMode + self.connectedToInternet =0 + if (self.internet_on()==True): +# sys.stdout.write('internet connection active \n') +# sys.stdout.flush() + self.connectedToInternet=1 + else: + self.connectedToInternet=0 + + def gaAddAction(self,Event,action,label,value): + self.gaAddObject(GA_EVENT_ObjClass(self.UserID,self.GoogleID,Event,action,label,value,self.appName,self.appVer)) + + def gaAddException(self,ExceptionName,ExceptionFatal): + self.gaAddObject(GA_EXCEPTION_ObjClass(self.UserID,self.GoogleID,ExceptionName,ExceptionFatal,self.appName,self.appVer)) + + def gaAddObject(self,Object): + if self.BlockingMode==1: + while self.GA_q.full(): + sleep(self.Timeout) +# sys.stdout.write('blocking mode=1 \n queue full - sleeping for timeout \n') # within Timout, the thread will empty part of the queue +# sys.stdout.flush() + lockState = self.threadLock.acquire(self.BlockingMode) + if lockState==1: +# sys.stdout.write('got lock, adding item \n') +# sys.stdout.flush() + try: + self.GA_q.put_nowait(Object) +# sys.stdout.write('got lock, item added \n') +# sys.stdout.flush() + except Queue.Full: +# sys.stdout.write('Queue full \n') +# sys.stdout.flush() + pass + self.threadLock.release() + + def emptyQueueToList(self,obj_list): + items=0 + while ((not self.GA_q.empty()) and (items<20)): + obj_list.append(self.GA_q.get_nowait().payload) + items+=1 +# print items + + def reportBatched(self,batched): + req = Request(url_batched, data=batched) + urlopen(req) + #requests.post(url_batched,data=batched) + + def emptyAndReportQ(self): + obj_list = [] + self.emptyQueueToList(obj_list) + if not len(obj_list): + return + batched = '\n'.join(obj_list) +# print batched # - for debug + self.reportBatched(batched) + + def printSelf(self): + print('remaining in queue:') + while not self.GA_q.empty(): + obj = self.GA_q.get_nowait() + print(obj.payload) + + def internet_on(self): + try: + urlopen(url_conn,timeout=10) + return True + except URLError as err: pass + return False + + def activate(self): + if (self.UserPermission==1): + self.thread.start() + + + +#***************************************------TEST--------------************************************** + +if __name__ == '__main__': + g = GAmanager(GoogleID='UA-75220362-4',UserID="Foo",QueueSize=100,Timeout=5,UserPermission=1,BlockingMode=1,appName='TRex',appVer='1.11.232') #timeout in seconds +#for i in range(0,35,1): +#i = 42 + g.gaAddAction(Event='stl',action='stl/udp_1pkt_simple.py {packet_count:1000,packet_len:9000}',label='Boo',value=20) + #g.gaAddAction(Event='test',action='start',label='Boo1',value=20) + +#g.gaAddException('MEMFAULT',1) +#g.gaAddException('MEMFAULT',1) +#g.gaAddException('MEMFAULT',1) +#g.gaAddException('MEMFAULT',1) +#g.gaAddException('MEMFAULT',1) +#g.gaAddException('MEMFAULT',1) + g.emptyAndReportQ() +# g.printSelf() +#print g.payload +#print g.size + + + + +#g.activate() +#g.gaAddAction(Event='test',action='start',label='1',value='1') +#sys.stdout.write('element added \n') +#sys.stdout.flush() +#g.gaAddAction(Event='test',action='start',label='2',value='1') +#sys.stdout.write('element added \n') +#sys.stdout.flush() +#g.gaAddAction(Event='test',action='start',label='3',value='1') +#sys.stdout.write('element added \n') +#sys.stdout.flush() + +#testdata = "v=1&t=event&tid=UA-75220362-4&cid=2&ec=test&ea=testing&el=testpacket&ev=2" +#r = requests.post(url_debug,data=testdata) +#print r + +#thread1 = ga_Thread(1,g) +#thread1.start() +#thread1.join() +#for i in range(1,10,1): +# sys.stdout.write('yesh %d'% (i)) +# sys.stdout.flush() + + +# add timing mechanism - DONE +# add exception mechanism - DONE +# add version mechanism - DONE +# ask Itay for unique ID generation per user + + + + + + + + + + + -- cgit 1.2.3-korg