From 1a175f47f0cc3010af47b4b18d5b01724950f0a0 Mon Sep 17 00:00:00 2001 From: Yaroslav Brustinov Date: Thu, 17 Nov 2016 16:52:27 +0200 Subject: docs fixes: http->https on jquery address fix paths in performance job add separator between creating and using performance data Signed-off-by: Yaroslav Brustinov --- doc/AnalyticsWebReport.py | 24 ++++-- doc/TRexDataAnalysis.py | 200 ++++++++++++++++++++++---------------------- doc/trex_analytics.asciidoc | 8 +- doc/trex_toc.asciidoc | 2 +- doc/ws_main.py | 9 +- 5 files changed, 124 insertions(+), 119 deletions(-) (limited to 'doc') diff --git a/doc/AnalyticsWebReport.py b/doc/AnalyticsWebReport.py index 9bf186bd..182d8367 100755 --- a/doc/AnalyticsWebReport.py +++ b/doc/AnalyticsWebReport.py @@ -5,14 +5,20 @@ import TRexDataAnalysis as tr import time -def main(): - analytics = ac.initialize_analyticsreporting() - # print 'retrieving data from Google Analytics' - current_date = time.strftime("%Y-%m-%d") - response = ac.get_report(analytics, '2016-11-06', current_date) - ga_all_data_dict, setups = ac.export_to_tuples(response) - tr.create_all_data(ga_all_data_dict, setups, '2016-11-06', current_date, save_path=os.getcwd() + '/images/', - add_stats='yes') +def main(verbose = False): + if verbose: + print('Retrieving data from Google Analytics') + analytics = ac.initialize_analyticsreporting() + current_date = time.strftime("%Y-%m-%d") + response = ac.get_report(analytics, '2016-11-06', current_date) + ga_all_data_dict, setups = ac.export_to_tuples(response) + dest_path = os.path.join(os.getcwd(), 'build', 'images') + if verbose: + print('Saving data to %s' % dest_path) + tr.create_all_data(ga_all_data_dict, setups, '2016-11-06', current_date, save_path = dest_path, + add_stats='yes') + if verbose: + print('Done without errors.') if __name__ == "__main__": - main() + main() diff --git a/doc/TRexDataAnalysis.py b/doc/TRexDataAnalysis.py index 3561b0f1..8f991dc3 100755 --- a/doc/TRexDataAnalysis.py +++ b/doc/TRexDataAnalysis.py @@ -6,131 +6,129 @@ matplotlib.use('Agg') from matplotlib import pyplot as plt import os -PATH_FOR_GRAPHS = 'Z:/trex/trex-doc/images/' - def convert_dict_to_dframe(data, categories, index=''): - data_input = {} - for category in categories: - data_input[category] = data[category] - if index: - df = pd.DataFrame(data_input, index=data[index]) - else: - df = pd.DataFrame(data_input) - return df + data_input = {} + for category in categories: + data_input[category] = data[category] + if index: + df = pd.DataFrame(data_input, index=data[index]) + else: + df = pd.DataFrame(data_input) + return df def plot_bar_by_category(data_frame, category, index='', graph_name='graph.png', show='', gtitle='', save_path=''): - if index: - data_frame = data_frame.sort_index(by=index) - print data_frame[index] - else: - print data_frame - data_frame = pd.DataFrame(data_frame[category], columns=category).astype(float) - data_frame.plot(kind='bar') - plt.xticks(rotation='horizontal') - plt.title(gtitle) - if save_path: - plt.savefig(save_path + graph_name) - if show: - plt.show() - - -def generate_csv(data_frame, file_name, save_path=(os.getcwd() + "/")): - f = open(save_path + file_name, 'w') - data_frame.to_csv(f) - f.close() + if index: + data_frame = data_frame.sort_index(by=index) + print data_frame[index] + else: + print data_frame + data_frame = pd.DataFrame(data_frame[category], columns=category).astype(float) + data_frame.plot(kind='bar') + plt.xticks(rotation='horizontal') + plt.title(gtitle) + if save_path: + plt.savefig(os.path.join(save_path, graph_name)) + if show: + plt.show() + + +def generate_csv(data_frame, file_name, save_path=os.getcwd()): + f = open(os.path.join(save_path, file_name), 'w') + data_frame.to_csv(f) + f.close() # category is an array of category names that will appear as metrics def plot_bar_by_test_name(data_frame, test_name, category, graph_name='graph.png', show='', gtitle='', save_path=''): - data_frame = data_frame[data_frame['Test_name'] == test_name] - plot_bar_by_category(data_frame, category, 'Test_name', graph_name, show, gtitle=test_name, save_path=save_path) + data_frame = data_frame[data_frame['Test_name'] == test_name] + plot_bar_by_category(data_frame, category, 'Test_name', graph_name, show, gtitle=test_name, save_path=save_path) def generate_dframe_for_test(test_name, test_data): - test_results = [] - test_mins = set() - test_maxs = set() - for query in test_data: - test_results.append(float(query[3])) - test_mins.add(float(query[4])) - test_maxs.add(float(query[5])) - df = pd.DataFrame({test_name: test_results}) - stats = tuple([float(df.mean()), min(test_mins), max(test_maxs)]) # stats = (avg_mpps,min,max) - return df, stats + test_results = [] + test_mins = set() + test_maxs = set() + for query in test_data: + test_results.append(float(query[3])) + test_mins.add(float(query[4])) + test_maxs.add(float(query[5])) + df = pd.DataFrame({test_name: test_results}) + stats = tuple([float(df.mean()), min(test_mins), max(test_maxs)]) # stats = (avg_mpps,min,max) + return df, stats def generate_dframe_arr_and_stats_of_tests_per_setup(date, setup_name, setup_dict): - dframe_arr_trend = [] - stats_arr = [] - dframe_arr_latest = [] - test_names = setup_dict.keys() - for test in test_names: - df, stats = generate_dframe_for_test(test, setup_dict[test]) - dframe_arr_trend.append(df) - stats_arr.append(stats) - df_latest = float(setup_dict[test][-1][3]) - dframe_arr_latest.append(df_latest) - dframe_arr_latest = pd.DataFrame({'Date': [date] * len(dframe_arr_latest), - 'Setup': [setup_name], - 'Test Name': test_names, - 'MPPS': dframe_arr_latest}, - index=range(1, len(dframe_arr_latest) + 1)) - stats_df = pd.DataFrame(stats_arr, index=setup_dict.keys(), columns=['Avg MPPS', 'Golden Min', 'Golden Max']) - stats_df.index.name = 'Test Name' - return dframe_arr_trend, stats_df, dframe_arr_latest + dframe_arr_trend = [] + stats_arr = [] + dframe_arr_latest = [] + test_names = setup_dict.keys() + for test in test_names: + df, stats = generate_dframe_for_test(test, setup_dict[test]) + dframe_arr_trend.append(df) + stats_arr.append(stats) + df_latest = float(setup_dict[test][-1][3]) + dframe_arr_latest.append(df_latest) + dframe_arr_latest = pd.DataFrame({'Date': [date] * len(dframe_arr_latest), + 'Setup': [setup_name], + 'Test Name': test_names, + 'MPPS': dframe_arr_latest}, + index=range(1, len(dframe_arr_latest) + 1)) + stats_df = pd.DataFrame(stats_arr, index=setup_dict.keys(), columns=['Avg MPPS', 'Golden Min', 'Golden Max']) + stats_df.index.name = 'Test Name' + return dframe_arr_trend, stats_df, dframe_arr_latest def create_plot_for_dframe_arr(dframe_arr, setup_name, start_date, end_date, show='no', save_path='', - file_name='trend_graph'): - dframe_all = pd.concat(dframe_arr, axis=1) - dframe_all = dframe_all.astype(float) - dframe_all.plot() - plt.legend(fontsize='small', loc='best') - plt.ylabel('MPPS') - plt.title('Setup: ' + setup_name) - plt.tick_params( - axis='x', - which='both', - bottom='off', - top='off', - labelbottom='off') - plt.xlabel('Time Period: ' + start_date + ' - ' + end_date) - if save_path: - plt.savefig(save_path + setup_name + file_name + '.png') - if show == 'yes': - plt.show() + file_name='trend_graph'): + dframe_all = pd.concat(dframe_arr, axis=1) + dframe_all = dframe_all.astype(float) + dframe_all.plot() + plt.legend(fontsize='small', loc='best') + plt.ylabel('MPPS') + plt.title('Setup: ' + setup_name) + plt.tick_params( + axis='x', + which='both', + bottom='off', + top='off', + labelbottom='off') + plt.xlabel('Time Period: ' + start_date + ' - ' + end_date) + if save_path: + plt.savefig(os.path.join(save_path, setup_name + file_name + '.png')) + if show == 'yes': + plt.show() def create_bar_plot_for_latest_runs_per_setup(dframe_all_tests_latest, setup_name, show='no', save_path=''): - plt.figure() - dframe_all_tests_latest['MPPS'].plot(kind='bar', legend=False) - dframe_all_tests_latest = dframe_all_tests_latest[['Test Name', 'Setup', 'Date', 'MPPS']] - plt.xticks(rotation='horizontal') - plt.xlabel('Index of Tests') - plt.ylabel('MPPS') - plt.title("Test Runs for Setup: " + setup_name) - if save_path: - plt.savefig(save_path + setup_name + '_latest_test_runs.png') - dframe_all_tests_latest = dframe_all_tests_latest.round(2) - dframe_all_tests_latest.to_csv(save_path + setup_name + '_latest_test_runs_stats.csv') - if show == 'yes': - plt.show() + plt.figure() + dframe_all_tests_latest['MPPS'].plot(kind='bar', legend=False) + dframe_all_tests_latest = dframe_all_tests_latest[['Test Name', 'Setup', 'Date', 'MPPS']] + plt.xticks(rotation='horizontal') + plt.xlabel('Index of Tests') + plt.ylabel('MPPS') + plt.title("Test Runs for Setup: " + setup_name) + if save_path: + plt.savefig(os.path.join(save_path, setup_name + '_latest_test_runs.png')) + dframe_all_tests_latest = dframe_all_tests_latest.round(2) + dframe_all_tests_latest.to_csv(os.path.join(save_path, setup_name + '_latest_test_runs_stats.csv')) + if show == 'yes': + plt.show() def create_all_data_per_setup(setup_dict, setup_name, start_date, end_date, show='no', save_path='', add_stats=''): - dframe_arr, stats_arr, dframe_latest_arr = generate_dframe_arr_and_stats_of_tests_per_setup(end_date, setup_name, - setup_dict) - create_bar_plot_for_latest_runs_per_setup(dframe_latest_arr, setup_name, show=show, save_path=save_path) - create_plot_for_dframe_arr(dframe_arr, setup_name, start_date, end_date, show, save_path) - if add_stats: - stats_arr = stats_arr.round(2) - stats_arr.to_csv(save_path + setup_name + '_trend_stats.csv') - plt.close('all') + dframe_arr, stats_arr, dframe_latest_arr = generate_dframe_arr_and_stats_of_tests_per_setup(end_date, setup_name, + setup_dict) + create_bar_plot_for_latest_runs_per_setup(dframe_latest_arr, setup_name, show=show, save_path=save_path) + create_plot_for_dframe_arr(dframe_arr, setup_name, start_date, end_date, show, save_path) + if add_stats: + stats_arr = stats_arr.round(2) + stats_arr.to_csv(os.path.join(save_path, setup_name + '_trend_stats.csv')) + plt.close('all') def create_all_data(ga_data, setup_names, start_date, end_date, save_path='', add_stats=''): - for setup_name in setup_names: - create_all_data_per_setup(ga_data[setup_name], setup_name, start_date, end_date, show='no', save_path=save_path, - add_stats=add_stats) + for setup_name in setup_names: + create_all_data_per_setup(ga_data[setup_name], setup_name, start_date, end_date, show='no', save_path=save_path, + add_stats=add_stats) diff --git a/doc/trex_analytics.asciidoc b/doc/trex_analytics.asciidoc index 84946dfe..db39a319 100755 --- a/doc/trex_analytics.asciidoc +++ b/doc/trex_analytics.asciidoc @@ -35,7 +35,7 @@ image:images/trex09_latest_test_runs.png[title="trex09 test runs",align="left",w [format="csv", options="header",halign='center'] |=== -include::images/trex09_latest_test_runs_stats.csv[] +include::build/images/trex09_latest_test_runs_stats.csv[] |=== === Trend: Analysis Over Time @@ -44,7 +44,7 @@ image:images/trex09trend_graph.png[title="trex09trend_graph",align="left",width= [format="csv", options="header",halign='center'] |=== -include::images/trex09_trend_stats.csv[] +include::build/images/trex09_trend_stats.csv[] |=== == Setup: Kiwi02 @@ -58,7 +58,7 @@ image:images/kiwi02_latest_test_runs.png[title="trex09 test runs",align="left",w [format="csv", options="header",halign='center'] |=== -include::images/kiwi02_latest_test_runs_stats.csv[] +include::build/images/kiwi02_latest_test_runs_stats.csv[] |=== === Trend: Analysis Over Time @@ -67,7 +67,7 @@ image:images/kiwi02trend_graph.png[title="kiwi02trend_graph",align="left",width= [format="csv", options="header",halign='center'] |=== -include::images/kiwi02_trend_stats.csv[] +include::build/images/kiwi02_trend_stats.csv[] |=== diff --git a/doc/trex_toc.asciidoc b/doc/trex_toc.asciidoc index a41d707f..eecb883f 100644 --- a/doc/trex_toc.asciidoc +++ b/doc/trex_toc.asciidoc @@ -24,7 +24,7 @@ ifdef::backend-xhtml11[] - +