diff options
Diffstat (limited to 'csit.infra.dash')
21 files changed, 376 insertions, 238 deletions
diff --git a/csit.infra.dash/app/.ebextensions/env-flask.config b/csit.infra.dash/app/.ebextensions/env-flask.config index f7ae7c500c..795aa82cdb 100644 --- a/csit.infra.dash/app/.ebextensions/env-flask.config +++ b/csit.infra.dash/app/.ebextensions/env-flask.config @@ -1,4 +1,18 @@ option_settings: aws:elasticbeanstalk:application:environment: FLASK_DEBUG: 0 - FLASK_ENV: production
\ No newline at end of file + FLASK_ENV: production + CSIT_START_TRENDING: "True" + CSIT_START_REPORT: "True" + CSIT_START_COMPARISONS: "True" + CSIT_START_COVERAGE: "True" + CSIT_START_STATISTICS: "True" + CSIT_START_FAILURES: "True" + CSIT_START_SEARCH: "True" + CSIT_START_DOC: "True" + CSIT_TITLE: "FD.io CSIT" + CSIT_BRAND: "CSIT-Dash" + CSIT_URL_CICD: "https://jenkins.fd.io/job/" + CSIT_URL_LOGS: "https://logs.fd.io/vex-yul-rot-jenkins-1/" + CSIT_URL_DOC: "https://csit.fd.io/cdocs/" + CSIT_TIME_PERIOD: 250
\ No newline at end of file diff --git a/csit.infra.dash/app/cdash/__init__.py b/csit.infra.dash/app/cdash/__init__.py index 3d3f2001a3..bf761da9ee 100644 --- a/csit.infra.dash/app/cdash/__init__.py +++ b/csit.infra.dash/app/cdash/__init__.py @@ -27,6 +27,7 @@ from .data.data import Data def init_app(): """Construct core Flask application with embedded Dash app. """ + logging.basicConfig( format=C.LOG_FORMAT, datefmt=C.LOG_DATE_FORMAT, @@ -67,76 +68,59 @@ def init_app(): ).read_all_data(days=time_period) # Import Dash applications. + err_msg = "Application not loaded, no data available." logging.info("\n\nStarting the applications:\n" + "-" * 26 + "\n") - if data["statistics"].empty or data["trending"].empty: - logging.error( - f'"{C.NEWS_TITLE}" application not loaded, no data available.' - ) - logging.error( - f'"{C.STATS_TITLE}" application not loaded, no data available.' - ) - else: - logging.info(C.NEWS_TITLE) - from .news.news import init_news - app = init_news( - app, - data_stats=data["statistics"], - data_trending=data["trending"] - ) + if C.START_FAILURES: + logging.info(C.NEWS_TITLE) + if data["statistics"].empty or data["trending"].empty: + logging.error(err_msg) + else: + from .news.news import init_news + app = init_news(app, data["statistics"], data["trending"]) + if C.START_STATISTICS: logging.info(C.STATS_TITLE) - from .stats.stats import init_stats - app = init_stats( - app, - data_stats=data["statistics"], - data_trending=data["trending"] - ) - - if data["trending"].empty: - logging.error( - f'"{C.TREND_TITLE}" application not loaded, no data available.' - ) - else: + if data["statistics"].empty or data["trending"].empty: + logging.error(err_msg) + else: + from .stats.stats import init_stats + app = init_stats(app, data["statistics"], data["trending"]) + if C.START_TRENDING: logging.info(C.TREND_TITLE) - from .trending.trending import init_trending - app = init_trending(app, data_trending=data["trending"]) - - if data["iterative"].empty: - logging.error( - f'"{C.REPORT_TITLE}" application not loaded, no data available.' - ) - logging.error( - f'"{C.COMP_TITLE}" application not loaded, no data available.' - ) - else: + if data["trending"].empty: + logging.error(err_msg) + else: + from .trending.trending import init_trending + app = init_trending(app, data["trending"]) + if C.START_REPORT: logging.info(C.REPORT_TITLE) - from .report.report import init_report - app = init_report(app, data_iterative=data["iterative"]) - + if data["iterative"].empty: + logging.error(err_msg) + else: + from .report.report import init_report + app = init_report(app, data["iterative"]) + if C.START_COMPARISONS: logging.info(C.COMP_TITLE) - from .comparisons.comparisons import init_comparisons - app = init_comparisons(app, data_iterative=data["iterative"]) - - if data["coverage"].empty: - logging.error(( - f'"{C.COVERAGE_TITLE}" application not loaded, ' - 'no data available.' - )) - else: + if data["iterative"].empty: + logging.error(err_msg) + else: + from .comparisons.comparisons import init_comparisons + app = init_comparisons(app, data["iterative"]) + if C.START_COVERAGE: logging.info(C.COVERAGE_TITLE) - from .coverage.coverage import init_coverage - app = init_coverage(app, data_coverage=data["coverage"]) - - if all((data["trending"].empty, data["iterative"].empty, - data["coverage"].empty)): - logging.error(( - f'"{C.SEARCH_TITLE}" application not loaded, ' - 'no data available.' - )) - else: + if data["coverage"].empty: + logging.error(err_msg) + else: + from .coverage.coverage import init_coverage + app = init_coverage(app, data["coverage"]) + if C.START_SEARCH: logging.info(C.SEARCH_TITLE) - from .search.search import init_search - app = init_search(app, data) + if all((data["trending"].empty, data["iterative"].empty, + data["coverage"].empty)): + logging.error(err_msg) + else: + from .search.search import init_search + app = init_search(app, data) return app diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py index 0e32f38b6c..5f6ba2db2a 100644 --- a/csit.infra.dash/app/cdash/comparisons/tables.py +++ b/csit.infra.dash/app/cdash/comparisons/tables.py @@ -315,11 +315,13 @@ def comparison_table( else: c_row = c_data[c_data["name"] == row["name"]] if not c_row.empty: - unit.add(f"{s_unit_factor}{row['unit']}") r_mean = row["mean"] r_std = row["stdev"] c_mean = c_row["mean"].values[0] c_std = c_row["stdev"].values[0] + if r_mean == 0.0 or c_mean == 0.0: + break + unit.add(f"{s_unit_factor}{row['unit']}") l_name.append(row["name"]) l_r_mean.append(r_mean / unit_factor) l_r_std.append(r_std / unit_factor) diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2306_device b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2406_device Binary files differindex f619ce8a8e..f619ce8a8e 100644 --- a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2306_device +++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2406_device diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2306_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2406_ndrpdr Binary files differindex 06bc618bea..06bc618bea 100644 --- a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2306_ndrpdr +++ b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2406_ndrpdr diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_hoststack Binary files differdeleted file mode 100644 index 993d16c18c..0000000000 --- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_hoststack +++ /dev/null diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_mrr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_mrr Binary files differdeleted file mode 100644 index 96832850b1..0000000000 --- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_mrr +++ /dev/null diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_ndrpdr Binary files differdeleted file mode 100644 index 2291bb8349..0000000000 --- a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2306_ndrpdr +++ /dev/null diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_hoststack Binary files differnew file mode 100644 index 0000000000..1e9c708253 --- /dev/null +++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_hoststack diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_mrr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_mrr Binary files differnew file mode 100644 index 0000000000..416679acdb --- /dev/null +++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_mrr diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_ndrpdr Binary files differnew file mode 100644 index 0000000000..e76e6ab8e5 --- /dev/null +++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_ndrpdr diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_soak b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_soak Binary files differnew file mode 100644 index 0000000000..4502ca4f59 --- /dev/null +++ b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2406_soak diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml index ed5fc0b269..11c5a04a50 100644 --- a/csit.infra.dash/app/cdash/data/data.yaml +++ b/csit.infra.dash/app/cdash/data/data.yaml @@ -111,9 +111,9 @@ - data_type: iterative partition: test_type partition_name: mrr - release: rls2306 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2306 - schema: iterative_rls2306_mrr + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 + schema: iterative_rls2310_mrr columns: - job - build @@ -130,9 +130,9 @@ - data_type: iterative partition: test_type partition_name: ndrpdr - release: rls2306 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2306 - schema: iterative_rls2306_ndrpdr + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 + schema: iterative_rls2310_ndrpdr columns: - job - build @@ -146,6 +146,10 @@ - result_pdr_lower_rate_value - result_ndr_lower_rate_unit - result_ndr_lower_rate_value + - result_pdr_lower_bandwidth_unit + - result_pdr_lower_bandwidth_value + - result_ndr_lower_bandwidth_unit + - result_ndr_lower_bandwidth_value - result_latency_reverse_pdr_90_hdrh - result_latency_reverse_pdr_50_hdrh - result_latency_reverse_pdr_10_hdrh @@ -159,9 +163,9 @@ - data_type: iterative partition: test_type partition_name: hoststack - release: rls2306 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2306 - schema: iterative_rls2306_hoststack + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 + schema: iterative_rls2310_hoststack columns: - job - build @@ -179,9 +183,9 @@ - data_type: coverage partition: test_type partition_name: ndrpdr - release: rls2306 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2306 - schema: coverage_rls2306_ndrpdr + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 + schema: coverage_rls2310_ndrpdr columns: - job - build @@ -205,10 +209,28 @@ - result_latency_forward_pdr_10_hdrh - data_type: coverage partition: test_type + partition_name: mrr + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 + schema: iterative_rls2310_mrr + columns: + - job + - build + - dut_type + - dut_version + - start_time + - passed + - test_id + - version + - result_receive_rate_rate_avg + - result_receive_rate_rate_stdev + - result_receive_rate_rate_unit +- data_type: coverage + partition: test_type partition_name: device - release: rls2306 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2306 - schema: coverage_rls2306_device + release: rls2310 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 + schema: coverage_rls2310_device columns: - job - build @@ -220,14 +242,15 @@ - data_type: iterative partition: test_type partition_name: mrr - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 - schema: iterative_rls2310_mrr + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 + schema: iterative_rls2402_mrr columns: - job - build - dut_type - dut_version + # - hosts - start_time - passed - test_id @@ -236,17 +259,22 @@ - result_receive_rate_rate_stdev - result_receive_rate_rate_unit - result_receive_rate_rate_values + - result_receive_rate_bandwidth_avg + - result_receive_rate_bandwidth_stdev + - result_receive_rate_bandwidth_unit + - result_receive_rate_bandwidth_values - data_type: iterative partition: test_type partition_name: ndrpdr - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 - schema: iterative_rls2310_ndrpdr + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 + schema: iterative_rls2402_ndrpdr columns: - job - build - dut_type - dut_version + # - hosts - start_time - passed - test_id @@ -272,14 +300,15 @@ - data_type: iterative partition: test_type partition_name: hoststack - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2310 - schema: iterative_rls2310_hoststack + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 + schema: iterative_rls2402_hoststack columns: - job - build - dut_type - dut_version + - hosts - tg_type - result_bandwidth_unit - result_bandwidth_value @@ -292,9 +321,9 @@ - data_type: coverage partition: test_type partition_name: ndrpdr - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 - schema: coverage_rls2310_ndrpdr + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 + schema: coverage_rls2402_ndrpdr columns: - job - build @@ -319,9 +348,9 @@ - data_type: coverage partition: test_type partition_name: mrr - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 - schema: iterative_rls2310_mrr + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 + schema: iterative_rls2402_mrr columns: - job - build @@ -337,9 +366,9 @@ - data_type: coverage partition: test_type partition_name: device - release: rls2310 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2310 - schema: coverage_rls2310_device + release: rls2402 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 + schema: coverage_rls2402_device columns: - job - build @@ -351,9 +380,9 @@ - data_type: iterative partition: test_type partition_name: mrr - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 - schema: iterative_rls2402_mrr + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2406 + schema: iterative_rls2406_mrr columns: - job - build @@ -375,9 +404,9 @@ - data_type: iterative partition: test_type partition_name: ndrpdr - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 - schema: iterative_rls2402_ndrpdr + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2406 + schema: iterative_rls2406_ndrpdr columns: - job - build @@ -409,9 +438,9 @@ - data_type: iterative partition: test_type partition_name: hoststack - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2402 - schema: iterative_rls2402_hoststack + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2406 + schema: iterative_rls2406_hoststack columns: - job - build @@ -427,12 +456,34 @@ - passed - test_id - version +- data_type: iterative + partition: test_type + partition_name: soak + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/iterative_rls2406 + schema: iterative_rls2406_soak + columns: + - job + - build + - dut_type + - dut_version + - hosts + - tg_type + - result_critical_rate_lower_bandwidth_unit + - result_critical_rate_lower_bandwidth_value + - result_critical_rate_lower_rate_unit + - result_critical_rate_lower_rate_value + - start_time + - passed + - telemetry + - test_id + - version - data_type: coverage partition: test_type partition_name: ndrpdr - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 - schema: coverage_rls2402_ndrpdr + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2406 + schema: coverage_rls2406_ndrpdr columns: - job - build @@ -457,9 +508,9 @@ - data_type: coverage partition: test_type partition_name: mrr - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 - schema: iterative_rls2402_mrr + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2406 + schema: iterative_rls2406_mrr columns: - job - build @@ -475,9 +526,9 @@ - data_type: coverage partition: test_type partition_name: device - release: rls2402 - path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2402 - schema: coverage_rls2402_device + release: rls2406 + path: s3://fdio-docs-s3-cloudfront-index/csit/parquet/coverage_rls2406 + schema: coverage_rls2406_device columns: - job - build diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py index 02202ea155..e13ec54a89 100644 --- a/csit.infra.dash/app/cdash/report/graphs.py +++ b/csit.infra.dash/app/cdash/report/graphs.py @@ -99,7 +99,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict, def get_y_values(data, y_data_max, param, norm_factor, release=str(), remove_outliers=False): if param == "result_receive_rate_rate_values": - if release == "rls2402": + if release in ("rls2402", "rls2406", "rls2410"): y_vals_raw = data["result_receive_rate_rate_avg"].to_list() else: y_vals_raw = data[param].to_list()[0] @@ -181,7 +181,7 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict, ) } - if itm["testtype"] == "mrr" and itm["rls"] in ("rls2306", "rls2310"): + if itm["testtype"] == "mrr" and itm["rls"] == "rls2310": trial_run = "trial" metadata["csit-ref"] = ( f"{itm_data['job'].to_list()[0]}/", diff --git a/csit.infra.dash/app/cdash/routes.py b/csit.infra.dash/app/cdash/routes.py index ed29fffa12..422bd1ab2e 100644 --- a/csit.infra.dash/app/cdash/routes.py +++ b/csit.infra.dash/app/cdash/routes.py @@ -24,15 +24,29 @@ from .utils.constants import Constants as C def home(): """Landing page. """ + + menu_itms = list() + if C.START_TRENDING: + menu_itms.append({"path": "/trending/", "title": C.TREND_TITLE}) + if C.START_REPORT: + menu_itms.append({"path": "/report/", "title": C.REPORT_TITLE}) + if C.START_COMPARISONS: + menu_itms.append({"path": "/comparisons/", "title": C.COMP_TITLE}) + if C.START_COVERAGE: + menu_itms.append({"path": "/coverage/", "title": C.COVERAGE_TITLE}) + if C.START_STATISTICS: + menu_itms.append({"path": "/stats/", "title": C.STATS_TITLE}) + if C.START_FAILURES: + menu_itms.append({"path": "/news/", "title": C.NEWS_TITLE}) + if C.START_SEARCH: + menu_itms.append({"path": "/search/", "title": C.SEARCH_TITLE}) + if C.START_DOC: + menu_itms.append({"path": "/cdocs/", "title": C.DOC_TITLE}) + return render_template( C.MAIN_HTML_LAYOUT_FILE, title=C.TITLE, + brand=C.BRAND, description=C.DESCRIPTION, - trending_title=C.TREND_TITLE, - report_title=C.REPORT_TITLE, - comp_title=C.COMP_TITLE, - stats_title=C.STATS_TITLE, - news_title=C.NEWS_TITLE, - cov_title=C.COVERAGE_TITLE, - search_title=C.SEARCH_TITLE + menu_itms=menu_itms ) diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py index 655c61c078..616a4028e6 100644 --- a/csit.infra.dash/app/cdash/stats/layout.py +++ b/csit.infra.dash/app/cdash/stats/layout.py @@ -196,7 +196,7 @@ class Layout: "dd-tbeds-value": self._default["tbed"], "al-job-children": html.A( self._default["job"], - href=f"{C.URL_JENKINS}{self._default['job']}", + href=f"{C.URL_CICD}{self._default['job']}", target="_blank" ) } @@ -631,7 +631,7 @@ class Layout: "al-job-children": html.A( self._default["job"], href=( - f"{C.URL_JENKINS}" + f"{C.URL_CICD}" f"{self._default['job']}" ), target="_blank" @@ -654,7 +654,7 @@ class Layout: { "al-job-children": html.A( job, - href=f"{C.URL_JENKINS}{job}", + href=f"{C.URL_CICD}{job}", target="_blank" ) } diff --git a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 index 7b0dadc5a0..97a71e358f 100644 --- a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 +++ b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 @@ -6,7 +6,7 @@ <header class="mb-auto"> <div> <h3 class="float-md-start mb-0 text-white"> - CSIT-Dash + {{ brand }} </h3> </div> </header> @@ -20,52 +20,18 @@ {{ description }} </p> <p class="lead"> - <p> - <a href="/trending/" class="btn btn-primary fw-bold w-25"> - {{ trending_title }} - </a> - </p> - <p> - <a href="/report/" class="btn btn-primary fw-bold w-25"> - {{ report_title }} - </a> - </p> - <p> - <a href="/comparisons/" class="btn btn-primary fw-bold w-25"> - {{ comp_title }} - </a> - </p> - <p> - <p> - <a href="/coverage/" class="btn btn-primary fw-bold w-25"> - {{ cov_title }} - </a> - </p> - <p> - <a href="/stats/" class="btn btn-primary fw-bold w-25"> - {{ stats_title }} - </a> - </p> - <p> - <a href="/news/" class="btn btn-primary fw-bold w-25"> - {{ news_title }} - </a> - </p> - <p> - <a href="/search/" class="btn btn-primary fw-bold w-25"> - {{ search_title }} - </a> - </p> - <p> - <a href="/cdocs/" class="btn btn-primary fw-bold w-25"> - Documentation - </a> - </p> + {% for itm in menu_itms %} + <p> + <a href="{{ itm['path'] }}" class="btn btn-primary fw-bold w-25"> + {{ itm['title'] }} + </a> + </p> + {% endfor %} </p> </main> <footer class="mt-auto text-white-50"> - <p>Copyright © 2016-2023 <a href="https://fd.io" class="text-white">The Fast Data Project</a>, a series of LF Projects, LLC.</p> + <p>Copyright © 2016-2024 <a href="https://fd.io" class="text-white">The Fast Data Project</a>, a series of LF Projects, LLC.</p> </footer> </div> {% endblock %} diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py index 840766488a..bafa7b7f42 100644 --- a/csit.infra.dash/app/cdash/utils/constants.py +++ b/csit.infra.dash/app/cdash/utils/constants.py @@ -17,11 +17,75 @@ does not need to be hard coded here, but can be read from environment variables. """ +import os import logging from dash import html +def get_str_from_env(env_var_name: str, default_value: str) -> str: + """Attempt to read string from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + If environment variable exists, but is empty (and default is not), + empty string is returned. + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if the env var does not exist. + :type env_var_names: str + :type default_value: str + :returns: The value read, or default value. + :rtype: str + """ + prefix = "CSIT_" + env_str = os.environ.get(prefix + env_var_name, None) + if env_str is not None: + return env_str + return default_value + + +def get_int_from_env(env_var_name: str, default_value: int) -> int: + """Attempt to read int from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + String value is read, default is returned also if conversion fails. + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if read or conversion fails. + :type env_var_names: str + :type default_value: int + :returns: The value read, or default value. + :rtype: int + """ + try: + return int(get_str_from_env(env_var_name, str())) + except ValueError: + return default_value + + +def get_bool_from_env(env_var_name: str, default_value: bool) -> bool: + """Attempt to read bool from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if read or conversion fails. + :type env_var_names: str + :type default_value: bool + :returns: The value read, or default value. + :rtype: bool + """ + env_str = get_str_from_env(env_var_name, str()).lower() + if env_str in ("true", "yes", "y", "1"): + return True + elif env_str in ("false", "no", "n", "0"): + return False + else: + return default_value + + class Constants: """Constants used in CDash. """ @@ -29,14 +93,24 @@ class Constants: ############################################################################ # General, application wide constants. + # Select applications to start. + START_TRENDING = get_bool_from_env("START_TRENDING", True) + START_REPORT = get_bool_from_env("START_REPORT", True) + START_COMPARISONS = get_bool_from_env("START_COMPARISONS", True) + START_COVERAGE = get_bool_from_env("START_COVERAGE", True) + START_STATISTICS = get_bool_from_env("START_STATISTICS", True) + START_FAILURES = get_bool_from_env("START_FAILURES", True) + START_SEARCH = get_bool_from_env("START_SEARCH", True) + START_DOC = get_bool_from_env("START_DOC", True) + # Logging settings. LOG_LEVEL = logging.INFO LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s" LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S" # The application title. - TITLE = "FD.io CSIT" - BRAND = "CSIT-Dash" + TITLE = get_str_from_env("TITLE", "FD.io CSIT") + BRAND = get_str_from_env("BRAND", "CSIT-Dash") # The application description. DESCRIPTION = "Performance Dashboard" @@ -45,14 +119,17 @@ class Constants: EXTERNAL_STYLESHEETS = ["/static/dist/css/bootstrap.css", ] # URL to Jenkins - URL_JENKINS = "https://jenkins.fd.io/job/" + URL_CICD = get_str_from_env("URL_CICD", "https://jenkins.fd.io/job/") # URL to logs - URL_LOGS = "https://s3-logs.fd.io/vex-yul-rot-jenkins-1/" + URL_LOGS = get_str_from_env( + "URL_LOGS", "https://logs.fd.io/vex-yul-rot-jenkins-1/" + ) # URL to the documentation - URL_DOC_TRENDING = "https://csit.fd.io/cdocs/methodology/trending/analysis/" - URL_DOC_REL_NOTES = "https://csit.fd.io/cdocs/release_notes/current/" + URL_DOC = get_str_from_env("URL_DOC", "https://csit.fd.io/cdocs/") + URL_DOC_TRENDING = URL_DOC + "methodology/trending/analysis/" + URL_DOC_REL_NOTES = URL_DOC + "release_notes/current/" # Path and name of the file specifying the HTML layout of the dash # application. @@ -82,7 +159,7 @@ class Constants: # now back to the past. # TIME_PERIOD = None - means all data (max MAX_TIME_PERIOD days) is read. # TIME_PERIOD = MAX_TIME_PERIOD - is the default value - TIME_PERIOD = MAX_TIME_PERIOD # [days] + TIME_PERIOD = get_int_from_env("TIME_PERIOD", MAX_TIME_PERIOD) # [days] ############################################################################ # General, application wide, layout affecting constants. @@ -467,3 +544,9 @@ class Constants: SEARCH_DOWNLOAD_FILE_NAME = "search_data.csv" ############################################################################ + # Documentation. + + # The title. + DOC_TITLE = "Documentation" + + ############################################################################ diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py index 306b4f60d1..e203dfbccd 100644 --- a/csit.infra.dash/app/cdash/utils/utils.py +++ b/csit.infra.dash/app/cdash/utils/utils.py @@ -480,37 +480,42 @@ def navbar_trending(active: tuple): :returns: Navigation bar. :rtype: dbc.NavbarSimple """ + children = list() + if C.START_TRENDING: + children.append(dbc.NavItem(dbc.NavLink( + C.TREND_TITLE, + active=active[0], + external_link=True, + href="/trending" + ))) + if C.START_FAILURES: + children.append(dbc.NavItem(dbc.NavLink( + C.NEWS_TITLE, + active=active[1], + external_link=True, + href="/news" + ))) + if C.START_STATISTICS: + children.append(dbc.NavItem(dbc.NavLink( + C.STATS_TITLE, + active=active[2], + external_link=True, + href="/stats" + ))) + if C.START_SEARCH: + children.append(dbc.NavItem(dbc.NavLink( + C.SEARCH_TITLE, + active=active[3], + external_link=True, + href="/search" + ))) + if C.START_DOC: + children.append(dbc.NavItem(dbc.NavLink( + "Documentation", + id="btn-documentation", + ))) return dbc.NavbarSimple( - children=[ - dbc.NavItem(dbc.NavLink( - C.TREND_TITLE, - active=active[0], - external_link=True, - href="/trending" - )), - dbc.NavItem(dbc.NavLink( - C.NEWS_TITLE, - active=active[1], - external_link=True, - href="/news" - )), - dbc.NavItem(dbc.NavLink( - C.STATS_TITLE, - active=active[2], - external_link=True, - href="/stats" - )), - dbc.NavItem(dbc.NavLink( - C.SEARCH_TITLE, - active=active[3], - external_link=True, - href="/search" - )), - dbc.NavItem(dbc.NavLink( - "Documentation", - id="btn-documentation", - )) - ], + children=children, id="navbarsimple-main", brand=C.BRAND, brand_href="/", @@ -529,38 +534,43 @@ def navbar_report(active: tuple): :returns: Navigation bar. :rtype: dbc.NavbarSimple """ + children = list() + if C.START_REPORT: + children.append(dbc.NavItem(dbc.NavLink( + C.REPORT_TITLE, + active=active[0], + external_link=True, + href="/report" + ))) + if C.START_COMPARISONS: + children.append(dbc.NavItem(dbc.NavLink( + "Comparisons", + active=active[1], + external_link=True, + href="/comparisons" + ))) + if C.START_COVERAGE: + children.append(dbc.NavItem(dbc.NavLink( + "Coverage Data", + active=active[2], + external_link=True, + href="/coverage" + ))) + if C.START_SEARCH: + children.append(dbc.NavItem(dbc.NavLink( + C.SEARCH_TITLE, + active=active[3], + external_link=True, + href="/search" + ))) + if C.START_DOC: + children.append(dbc.NavItem(dbc.NavLink( + "Documentation", + id="btn-documentation", + ))) return dbc.NavbarSimple( + children=children, id="navbarsimple-main", - children=[ - dbc.NavItem(dbc.NavLink( - C.REPORT_TITLE, - active=active[0], - external_link=True, - href="/report" - )), - dbc.NavItem(dbc.NavLink( - "Comparisons", - active=active[1], - external_link=True, - href="/comparisons" - )), - dbc.NavItem(dbc.NavLink( - "Coverage Data", - active=active[2], - external_link=True, - href="/coverage" - )), - dbc.NavItem(dbc.NavLink( - C.SEARCH_TITLE, - active=active[3], - external_link=True, - href="/search" - )), - dbc.NavItem(dbc.NavLink( - "Documentation", - id="btn-documentation", - )) - ], brand=C.BRAND, brand_href="/", brand_external_link=True, diff --git a/csit.infra.dash/do_run.sh b/csit.infra.dash/do_run.sh index c87e434416..9ededc77a2 100755 --- a/csit.infra.dash/do_run.sh +++ b/csit.infra.dash/do_run.sh @@ -7,4 +7,4 @@ command -v docker || exit 1 export UID=$(id -u) export GID=$(id -g) -docker compose up --remove-orphans +docker compose up --remove-orphans --detach diff --git a/csit.infra.dash/docker-compose.yaml b/csit.infra.dash/docker-compose.yaml index 5ed71345fb..a3b5e1a683 100644 --- a/csit.infra.dash/docker-compose.yaml +++ b/csit.infra.dash/docker-compose.yaml @@ -6,6 +6,20 @@ services: environment: FLASK_DEBUG: 1 FLASK_ENV: "development" + CSIT_START_TRENDING: "True" + CSIT_START_REPORT: "True" + CSIT_START_COMPARISONS: "True" + CSIT_START_COVERAGE: "True" + CSIT_START_STATISTICS: "True" + CSIT_START_FAILURES: "True" + CSIT_START_SEARCH: "True" + CSIT_START_DOC: "True" + CSIT_TITLE: "FD.io CSIT" + CSIT_BRAND: "CSIT-Dash" + CSIT_URL_CICD: "https://jenkins.fd.io/job/" + CSIT_URL_LOGS: "https://logs.fd.io/vex-yul-rot-jenkins-1/" + CSIT_URL_DOC: "https://csit.fd.io/cdocs/" + CSIT_TIME_PERIOD: 250 mem_limit: "16g" ports: - "5000:5000" |