fixed issue with fetching elk data - now fetching from trex-perf-* (with wildcard)

Signed-off-by: itraviv <itraviv@cisco.com>

3 files changed, 13 insertions, 12 deletions

diff --git a/doc/AnalyticsWebReport.py b/doc/AnalyticsWebReport.py
index ed35d86c..89c7771a 100755
--- a/doc/AnalyticsWebReport.py
+++ b/doc/AnalyticsWebReport.py
@@ -18,7 +18,7 @@ def main(verbose=False, source='elk', detailed_test_stats='yes'):
         response = ac.get_report(analytics, start_date, current_date)
         all_data_dict, setups = ac.export_to_tuples(response)
     if source == 'elk':
-        elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-000004', port=9200)
+        elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-*', port=9200)
         all_data_dict = elk_manager.fetch_and_parse()
     dest_path = os.path.join(os.getcwd(), 'build', 'images')
     if verbose:
diff --git a/doc/ELKConnect.py b/doc/ELKConnect.py
index fb7e2c29..fe3e121f 100755
--- a/doc/ELKConnect.py
+++ b/doc/ELKConnect.py
@@ -18,7 +18,7 @@ import elasticsearch.helpers
 

 

 class ELKManager:

-    def __init__(self, hostname, index='trex_perf-000004', port=9200):

+    def __init__(self, hostname, index='trex_perf-*', port=9200):

         self.hostname = hostname

         self.index = index

         self.port = port

@@ -29,14 +29,14 @@ class ELKManager:
 

     @staticmethod

     def time_res_calculation():

-        milli_since_epoch = int(time.time() * 1000)

+        seconds_since_epoch = int(time.time())

         time_2w_ago = datetime.date.timetuple(datetime.datetime.utcnow() - datetime.timedelta(weeks=2))

-        two_w_ago_epoch_milli = int(time.mktime(time_2w_ago) * 1000)

-        return milli_since_epoch, two_w_ago_epoch_milli

+        two_w_ago_epoch_second = int(time.mktime(time_2w_ago))

+        return seconds_since_epoch, two_w_ago_epoch_second

 

     def fetch_all_data(self):

         res = {}

-        milli_since_epoch, two_weeks_ago_epoch_milli = self.time_res_calculation()

+        seconds_since_epoch, two_w_ago_epoch_second = self.time_res_calculation()

         for setup_name in self.setup_names:

             query = {

                 "_source": ["info.setup.name", "test.name", "test.mpps_pc", "timestamp", "build_id"],

@@ -44,15 +44,15 @@ class ELKManager:
                 "query": {

                     "bool": {

                         "filter": [

-                            {"range": {

-                                "timestamp": {"gte": two_weeks_ago_epoch_milli, "lte": milli_since_epoch,

-                                              "format": "epoch_millis"}}},

                             {"term": {"info.setup.name": setup_name}},

-                            {"term": {"test.type": "stateless"}}

+                            {"term": {"test.type": "stateless"}},

+                            {"range": {"timestamp": {"gte": two_w_ago_epoch_second, "lte": seconds_since_epoch,

+                                                     "format": "epoch_second"}}}

                         ]

                     }

                 }

             }

+

             res[setup_name] = list(elasticsearch.helpers.scan(self.es, index=self.index, query=query, size=10000))

         self.all_data_raw = res

 

diff --git a/doc/TRexDataAnalysis.py b/doc/TRexDataAnalysis.py
index 5a2966c9..9aeba0ec 100755
--- a/doc/TRexDataAnalysis.py
+++ b/doc/TRexDataAnalysis.py
@@ -225,8 +225,9 @@ def create_all_data(ga_data, end_date, save_path='', detailed_test_stats=''):
     if detailed_test_stats:
         if os.path.exists(os.path.join(save_path, '_detailed_table.csv')):
             os.remove(os.path.join(save_path, '_detailed_table.csv'))
-        all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
-        all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
+        if all_setups_data:
+            all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
+            all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
 
     trex07setup = all_setups['trex07']
     trex08setup = all_setups['trex08']