From 44972cd90712cd372af59938e6ec477a338dd120 Mon Sep 17 00:00:00 2001 From: itraviv Date: Thu, 23 Mar 2017 13:45:49 +0200 Subject: fixed issue with fetching elk data - now fetching from trex-perf-* (with wildcard) Signed-off-by: itraviv --- doc/AnalyticsWebReport.py | 2 +- doc/ELKConnect.py | 18 +++++++++--------- doc/TRexDataAnalysis.py | 5 +++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/AnalyticsWebReport.py b/doc/AnalyticsWebReport.py index ed35d86c..89c7771a 100755 --- a/doc/AnalyticsWebReport.py +++ b/doc/AnalyticsWebReport.py @@ -18,7 +18,7 @@ def main(verbose=False, source='elk', detailed_test_stats='yes'): response = ac.get_report(analytics, start_date, current_date) all_data_dict, setups = ac.export_to_tuples(response) if source == 'elk': - elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-000004', port=9200) + elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-*', port=9200) all_data_dict = elk_manager.fetch_and_parse() dest_path = os.path.join(os.getcwd(), 'build', 'images') if verbose: diff --git a/doc/ELKConnect.py b/doc/ELKConnect.py index fb7e2c29..fe3e121f 100755 --- a/doc/ELKConnect.py +++ b/doc/ELKConnect.py @@ -18,7 +18,7 @@ import elasticsearch.helpers class ELKManager: - def __init__(self, hostname, index='trex_perf-000004', port=9200): + def __init__(self, hostname, index='trex_perf-*', port=9200): self.hostname = hostname self.index = index self.port = port @@ -29,14 +29,14 @@ class ELKManager: @staticmethod def time_res_calculation(): - milli_since_epoch = int(time.time() * 1000) + seconds_since_epoch = int(time.time()) time_2w_ago = datetime.date.timetuple(datetime.datetime.utcnow() - datetime.timedelta(weeks=2)) - two_w_ago_epoch_milli = int(time.mktime(time_2w_ago) * 1000) - return milli_since_epoch, two_w_ago_epoch_milli + two_w_ago_epoch_second = int(time.mktime(time_2w_ago)) + return seconds_since_epoch, two_w_ago_epoch_second def fetch_all_data(self): res = {} - milli_since_epoch, two_weeks_ago_epoch_milli = self.time_res_calculation() + seconds_since_epoch, two_w_ago_epoch_second = self.time_res_calculation() for setup_name in self.setup_names: query = { "_source": ["info.setup.name", "test.name", "test.mpps_pc", "timestamp", "build_id"], @@ -44,15 +44,15 @@ class ELKManager: "query": { "bool": { "filter": [ - {"range": { - "timestamp": {"gte": two_weeks_ago_epoch_milli, "lte": milli_since_epoch, - "format": "epoch_millis"}}}, {"term": {"info.setup.name": setup_name}}, - {"term": {"test.type": "stateless"}} + {"term": {"test.type": "stateless"}}, + {"range": {"timestamp": {"gte": two_w_ago_epoch_second, "lte": seconds_since_epoch, + "format": "epoch_second"}}} ] } } } + res[setup_name] = list(elasticsearch.helpers.scan(self.es, index=self.index, query=query, size=10000)) self.all_data_raw = res diff --git a/doc/TRexDataAnalysis.py b/doc/TRexDataAnalysis.py index 5a2966c9..9aeba0ec 100755 --- a/doc/TRexDataAnalysis.py +++ b/doc/TRexDataAnalysis.py @@ -225,8 +225,9 @@ def create_all_data(ga_data, end_date, save_path='', detailed_test_stats=''): if detailed_test_stats: if os.path.exists(os.path.join(save_path, '_detailed_table.csv')): os.remove(os.path.join(save_path, '_detailed_table.csv')) - all_setups_data_dframe = pd.DataFrame().append(all_setups_data) - all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv')) + if all_setups_data: + all_setups_data_dframe = pd.DataFrame().append(all_setups_data) + all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv')) trex07setup = all_setups['trex07'] trex08setup = all_setups['trex08'] -- cgit 1.2.3-korg