import os import outer_packages import json import pprint from elasticsearch import Elasticsearch from pprint import pprint from elasticsearch import helpers import random import datetime # one object example for perf def create_one_object (build_id): d={}; sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30)); info = {}; img={} img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"]) img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S") img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"]) img['formal'] = False setup={} setup['distro']='Ubunto14.03' setup['kernel']='2.6.12' setup['baremetal']=True setup['hypervisor']='None' setup['name']='trex07' setup['cpu-sockets']=2 setup['cores']=16 setup['cpu-speed']=3.5 setup['dut'] ='loopback' setup['drv-name']='mlx5' setup['nic-ports']=2 setup['total-nic-ports']=2 setup['nic-speed'] ="40GbE" info['image'] = img info['setup'] = setup d['info'] =info; d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S") d['build_id']=str("build-%d" %(build_id)) d['test']={ "name" : "test1", "type" : "stateless", "cores" : random.randint(1,10), "cpu%" : random.randint(60,99), "mpps" : random.randint(9,32), "mpps_pc" : random.randint(9,32), "gbps_pc" : random.randint(9,32), "gbps" : random.randint(9,32), "avg-pktsize" : random.randint(60,1500), "latecny" : { "min" : random.randint(1,10), "max" : random.randint(100,120), "avr" : random.randint(1,60) } }; return(d) class EsHelper(object): def __init__ (self, es, alias, index_name, mapping): self.es = es self.alias = alias self.index_name = index_name self.mapping = mapping self.setting = { "index.mapper.dynamic":"false"}; def delete (self): es=self.es; es.indices.delete(index=self.alias, ignore=[400, 404]); def is_exists (self): es=self.es; return es.indices.exists(index=self.alias, ignore=[400, 404]) def create_first_fime (self): es=self.es; index_name=self.index_name es.indices.create(index=index_name, ignore=[],body = { "aliases": { self.alias : {} }, "mappings" : { "data": self.mapping }, "settings" : self.setting }); def update(self): es=self.es; es.indices.put_mapping(index=self.alias, doc_type="data",body=self.mapping); es.indices.rollover(alias=self.alias,body={ "conditions": { "max_age": "30d", "max_docs": 100000 }, "mappings" : { "data": self.mapping }, "settings" : self.setting } ); def open(self): if not self.is_exists(): self.create_first_fime () else: self.update() def close(self): pass; def push_data(self,data): es=self.es; es.index(index=self.alias,doc_type="data", body=data); def create_reg_object (build_id): d={}; sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30)); info = {}; img={} img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"]) img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S") img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"]) img['formal'] = False setup={} setup['distro']='Ubunto14.03' setup['kernel']='2.6.12' setup['baremetal']=True setup['hypervisor']='None' setup['name']='trex07' setup['cpu-sockets']=2 setup['cores']=16 setup['cpu-speed']=3.5 setup['dut'] ='loopback' setup['drv-name']='mlx5' setup['nic-ports']=2 setup['total-nic-ports']=2 setup['nic-speed'] ="40GbE" info['image'] = img info['setup'] = setup d['info'] =info; d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S") d['build_id']=str("build-%d" %(build_id)) d['test']= { "name" : "stateful_tests.trex_imix_test.CTRexIMIX_Test.test_routing_imix" , "type" : "stateless", "duration_sec" : random.uniform(1,10), "result" : random.choice(["PASS","SKIP","FAIL"]), "stdout" : """ LATEST RESULT OBJECT: Total ARP received : 16 pkts maximum-latency : 300 usec average-latency : 277 usec latency-any-error : ERROR """ }; return(d) # how to add new keyword # you can add a new field but you can't remove old field class TRexEs(object): def __init__ (self, host, port, ): self.es = Elasticsearch([{"host": host, "port": port}]) es=self.es; res=es.info() es_version=res["version"]["number"]; l=es_version.split('.'); if not(len(l)==3 and int(l[0])>=5): print("NOT valid ES version should be at least 5.0.x",es_version); raise RuntimeError setup_info = { # constant per setup "properties": { "image" : { "properties": { "sha" : { "type": "keyword" }, # git sha "build_time" : { "type": "date", # build time "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"}, "version" : { "type": "keyword" }, # version name like 'v2.12' "formal" : { "type": "boolean" }, # true for formal release } }, "setup" : { "properties": { "distro" : { "type": "keyword" }, # 'ubuntu' "kernel" : { "type": "keyword" }, # 2.3.19 "baremetal" : { "type": "boolean" }, # true or false for "hypervisor" : { "type": "keyword" }, # kvm,esxi , none "name" : { "type": "keyword" }, # setup name , e.g. kiwi02 "cpu-sockets" : { "type": "long" }, # number of socket "cores" : { "type": "long" }, # total cores "cpu-speed" : { "type": "double" }, # 3.5 in ghz "dut" : { "type": "keyword" }, # asr1k, loopback "drv-name" : { "type": "keyword" }, # vic, mlx5,599,xl710,x710 "nic-ports" : { "type": "long" }, #2,1,4 "total-nic-ports" : { "type": "long" }, #8 "nic-speed" : { "type": "keyword" }, #40Gb } } } } perf_mapping = { "dynamic": "strict", "properties": { "scenario" : { "type": "keyword" }, "build_id" : { "type": "keyword" }, "timestamp" : { "type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"}, "info" : setup_info, "test" : { "properties": { "name" : { "type": "keyword" }, # name of the test "type" : { "type": "keyword" }, # stateless,stateful, other "cores" : { "type": "long" }, "cpu%" : { "type": "double" }, "mpps" : { "type": "double" }, "streams_count" : { "type": "long" }, "mpps_pc" : { "type": "double" }, "gbps_pc" : { "type": "double" }, "gbps" : { "type": "double" }, "avg-pktsize" : { "type": "long" }, "kcps" : { "type": "double" }, "latecny" : { "properties": { "min" : { "type": "double" }, "max" : { "type": "double" }, "avr" : { "type": "double" }, "max-win" : { "type": "double" }, "drop-rate" : { "type": "double" }, "jitter" : { "type": "double" }, } } } } } } self.perf = EsHelper(es=es, alias="perf", index_name='trex_perf-000001', mapping=perf_mapping) reg_mapping = { "dynamic": "strict", "properties": { "scenario" : { "type": "keyword" }, "build_id" : { "type": "keyword" }, "timestamp" : { "type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"}, "info" : setup_info, "test" : { "properties": { "name" : { "type" : "text" }, # name of the test "name_key" : { "type" : "keyword" }, # name of the test "name_full" : { "type" : "keyword" }, # full name of the test "type" : { "type" : "keyword" }, # stateless,stateful, other "duration_sec" : { "type": "double" }, # sec "result" : { "type" : "keyword" }, # PASS,FAIL,SKIP "stdout" : { "type" : "text" }, # output in case of faliue } } } } self.reg = EsHelper(es=es, alias="reg", index_name='trex_reg-000001', mapping=reg_mapping) self.perf.open(); self.reg.open();