summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoritraviv <itraviv@cisco.com>2017-03-23 13:45:49 +0200
committeritraviv <itraviv@cisco.com>2017-03-23 13:46:07 +0200
commit44972cd90712cd372af59938e6ec477a338dd120 (patch)
treee90c60d688b14eae478179ed61f7707ce11f37ca
parentbe08ffe2c6e662032115b2a77f096b98955e3143 (diff)
fixed issue with fetching elk data - now fetching from trex-perf-* (with wildcard)
Signed-off-by: itraviv <itraviv@cisco.com>
-rwxr-xr-xdoc/AnalyticsWebReport.py2
-rwxr-xr-xdoc/ELKConnect.py18
-rwxr-xr-xdoc/TRexDataAnalysis.py5
3 files changed, 13 insertions, 12 deletions
diff --git a/doc/AnalyticsWebReport.py b/doc/AnalyticsWebReport.py
index ed35d86c..89c7771a 100755
--- a/doc/AnalyticsWebReport.py
+++ b/doc/AnalyticsWebReport.py
@@ -18,7 +18,7 @@ def main(verbose=False, source='elk', detailed_test_stats='yes'):
response = ac.get_report(analytics, start_date, current_date)
all_data_dict, setups = ac.export_to_tuples(response)
if source == 'elk':
- elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-000004', port=9200)
+ elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-*', port=9200)
all_data_dict = elk_manager.fetch_and_parse()
dest_path = os.path.join(os.getcwd(), 'build', 'images')
if verbose:
diff --git a/doc/ELKConnect.py b/doc/ELKConnect.py
index fb7e2c29..fe3e121f 100755
--- a/doc/ELKConnect.py
+++ b/doc/ELKConnect.py
@@ -18,7 +18,7 @@ import elasticsearch.helpers
class ELKManager:
- def __init__(self, hostname, index='trex_perf-000004', port=9200):
+ def __init__(self, hostname, index='trex_perf-*', port=9200):
self.hostname = hostname
self.index = index
self.port = port
@@ -29,14 +29,14 @@ class ELKManager:
@staticmethod
def time_res_calculation():
- milli_since_epoch = int(time.time() * 1000)
+ seconds_since_epoch = int(time.time())
time_2w_ago = datetime.date.timetuple(datetime.datetime.utcnow() - datetime.timedelta(weeks=2))
- two_w_ago_epoch_milli = int(time.mktime(time_2w_ago) * 1000)
- return milli_since_epoch, two_w_ago_epoch_milli
+ two_w_ago_epoch_second = int(time.mktime(time_2w_ago))
+ return seconds_since_epoch, two_w_ago_epoch_second
def fetch_all_data(self):
res = {}
- milli_since_epoch, two_weeks_ago_epoch_milli = self.time_res_calculation()
+ seconds_since_epoch, two_w_ago_epoch_second = self.time_res_calculation()
for setup_name in self.setup_names:
query = {
"_source": ["info.setup.name", "test.name", "test.mpps_pc", "timestamp", "build_id"],
@@ -44,15 +44,15 @@ class ELKManager:
"query": {
"bool": {
"filter": [
- {"range": {
- "timestamp": {"gte": two_weeks_ago_epoch_milli, "lte": milli_since_epoch,
- "format": "epoch_millis"}}},
{"term": {"info.setup.name": setup_name}},
- {"term": {"test.type": "stateless"}}
+ {"term": {"test.type": "stateless"}},
+ {"range": {"timestamp": {"gte": two_w_ago_epoch_second, "lte": seconds_since_epoch,
+ "format": "epoch_second"}}}
]
}
}
}
+
res[setup_name] = list(elasticsearch.helpers.scan(self.es, index=self.index, query=query, size=10000))
self.all_data_raw = res
diff --git a/doc/TRexDataAnalysis.py b/doc/TRexDataAnalysis.py
index 5a2966c9..9aeba0ec 100755
--- a/doc/TRexDataAnalysis.py
+++ b/doc/TRexDataAnalysis.py
@@ -225,8 +225,9 @@ def create_all_data(ga_data, end_date, save_path='', detailed_test_stats=''):
if detailed_test_stats:
if os.path.exists(os.path.join(save_path, '_detailed_table.csv')):
os.remove(os.path.join(save_path, '_detailed_table.csv'))
- all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
- all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
+ if all_setups_data:
+ all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
+ all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
trex07setup = all_setups['trex07']
trex08setup = all_setups['trex08']