diff options
| -rw-r--r-- | csit.infra.dash/app/.ebextensions/env-flask.config | 16 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/__init__.py | 108 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/routes.py | 28 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/stats/layout.py | 6 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/templates/base_layout.jinja2 | 52 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/constants.py | 97 | ||||
| -rw-r--r-- | csit.infra.dash/app/cdash/utils/utils.py | 130 | ||||
| -rw-r--r-- | csit.infra.dash/docker-compose.yaml | 14 |
8 files changed, 268 insertions, 183 deletions
diff --git a/csit.infra.dash/app/.ebextensions/env-flask.config b/csit.infra.dash/app/.ebextensions/env-flask.config index f7ae7c500c..795aa82cdb 100644 --- a/csit.infra.dash/app/.ebextensions/env-flask.config +++ b/csit.infra.dash/app/.ebextensions/env-flask.config @@ -1,4 +1,18 @@ option_settings: aws:elasticbeanstalk:application:environment: FLASK_DEBUG: 0 - FLASK_ENV: production
\ No newline at end of file + FLASK_ENV: production + CSIT_START_TRENDING: "True" + CSIT_START_REPORT: "True" + CSIT_START_COMPARISONS: "True" + CSIT_START_COVERAGE: "True" + CSIT_START_STATISTICS: "True" + CSIT_START_FAILURES: "True" + CSIT_START_SEARCH: "True" + CSIT_START_DOC: "True" + CSIT_TITLE: "FD.io CSIT" + CSIT_BRAND: "CSIT-Dash" + CSIT_URL_CICD: "https://jenkins.fd.io/job/" + CSIT_URL_LOGS: "https://logs.fd.io/vex-yul-rot-jenkins-1/" + CSIT_URL_DOC: "https://csit.fd.io/cdocs/" + CSIT_TIME_PERIOD: 250
\ No newline at end of file diff --git a/csit.infra.dash/app/cdash/__init__.py b/csit.infra.dash/app/cdash/__init__.py index 3d3f2001a3..bf761da9ee 100644 --- a/csit.infra.dash/app/cdash/__init__.py +++ b/csit.infra.dash/app/cdash/__init__.py @@ -27,6 +27,7 @@ from .data.data import Data def init_app(): """Construct core Flask application with embedded Dash app. """ + logging.basicConfig( format=C.LOG_FORMAT, datefmt=C.LOG_DATE_FORMAT, @@ -67,76 +68,59 @@ def init_app(): ).read_all_data(days=time_period) # Import Dash applications. + err_msg = "Application not loaded, no data available." logging.info("\n\nStarting the applications:\n" + "-" * 26 + "\n") - if data["statistics"].empty or data["trending"].empty: - logging.error( - f'"{C.NEWS_TITLE}" application not loaded, no data available.' - ) - logging.error( - f'"{C.STATS_TITLE}" application not loaded, no data available.' - ) - else: - logging.info(C.NEWS_TITLE) - from .news.news import init_news - app = init_news( - app, - data_stats=data["statistics"], - data_trending=data["trending"] - ) + if C.START_FAILURES: + logging.info(C.NEWS_TITLE) + if data["statistics"].empty or data["trending"].empty: + logging.error(err_msg) + else: + from .news.news import init_news + app = init_news(app, data["statistics"], data["trending"]) + if C.START_STATISTICS: logging.info(C.STATS_TITLE) - from .stats.stats import init_stats - app = init_stats( - app, - data_stats=data["statistics"], - data_trending=data["trending"] - ) - - if data["trending"].empty: - logging.error( - f'"{C.TREND_TITLE}" application not loaded, no data available.' - ) - else: + if data["statistics"].empty or data["trending"].empty: + logging.error(err_msg) + else: + from .stats.stats import init_stats + app = init_stats(app, data["statistics"], data["trending"]) + if C.START_TRENDING: logging.info(C.TREND_TITLE) - from .trending.trending import init_trending - app = init_trending(app, data_trending=data["trending"]) - - if data["iterative"].empty: - logging.error( - f'"{C.REPORT_TITLE}" application not loaded, no data available.' - ) - logging.error( - f'"{C.COMP_TITLE}" application not loaded, no data available.' - ) - else: + if data["trending"].empty: + logging.error(err_msg) + else: + from .trending.trending import init_trending + app = init_trending(app, data["trending"]) + if C.START_REPORT: logging.info(C.REPORT_TITLE) - from .report.report import init_report - app = init_report(app, data_iterative=data["iterative"]) - + if data["iterative"].empty: + logging.error(err_msg) + else: + from .report.report import init_report + app = init_report(app, data["iterative"]) + if C.START_COMPARISONS: logging.info(C.COMP_TITLE) - from .comparisons.comparisons import init_comparisons - app = init_comparisons(app, data_iterative=data["iterative"]) - - if data["coverage"].empty: - logging.error(( - f'"{C.COVERAGE_TITLE}" application not loaded, ' - 'no data available.' - )) - else: + if data["iterative"].empty: + logging.error(err_msg) + else: + from .comparisons.comparisons import init_comparisons + app = init_comparisons(app, data["iterative"]) + if C.START_COVERAGE: logging.info(C.COVERAGE_TITLE) - from .coverage.coverage import init_coverage - app = init_coverage(app, data_coverage=data["coverage"]) - - if all((data["trending"].empty, data["iterative"].empty, - data["coverage"].empty)): - logging.error(( - f'"{C.SEARCH_TITLE}" application not loaded, ' - 'no data available.' - )) - else: + if data["coverage"].empty: + logging.error(err_msg) + else: + from .coverage.coverage import init_coverage + app = init_coverage(app, data["coverage"]) + if C.START_SEARCH: logging.info(C.SEARCH_TITLE) - from .search.search import init_search - app = init_search(app, data) + if all((data["trending"].empty, data["iterative"].empty, + data["coverage"].empty)): + logging.error(err_msg) + else: + from .search.search import init_search + app = init_search(app, data) return app diff --git a/csit.infra.dash/app/cdash/routes.py b/csit.infra.dash/app/cdash/routes.py index ed29fffa12..422bd1ab2e 100644 --- a/csit.infra.dash/app/cdash/routes.py +++ b/csit.infra.dash/app/cdash/routes.py @@ -24,15 +24,29 @@ from .utils.constants import Constants as C def home(): """Landing page. """ + + menu_itms = list() + if C.START_TRENDING: + menu_itms.append({"path": "/trending/", "title": C.TREND_TITLE}) + if C.START_REPORT: + menu_itms.append({"path": "/report/", "title": C.REPORT_TITLE}) + if C.START_COMPARISONS: + menu_itms.append({"path": "/comparisons/", "title": C.COMP_TITLE}) + if C.START_COVERAGE: + menu_itms.append({"path": "/coverage/", "title": C.COVERAGE_TITLE}) + if C.START_STATISTICS: + menu_itms.append({"path": "/stats/", "title": C.STATS_TITLE}) + if C.START_FAILURES: + menu_itms.append({"path": "/news/", "title": C.NEWS_TITLE}) + if C.START_SEARCH: + menu_itms.append({"path": "/search/", "title": C.SEARCH_TITLE}) + if C.START_DOC: + menu_itms.append({"path": "/cdocs/", "title": C.DOC_TITLE}) + return render_template( C.MAIN_HTML_LAYOUT_FILE, title=C.TITLE, + brand=C.BRAND, description=C.DESCRIPTION, - trending_title=C.TREND_TITLE, - report_title=C.REPORT_TITLE, - comp_title=C.COMP_TITLE, - stats_title=C.STATS_TITLE, - news_title=C.NEWS_TITLE, - cov_title=C.COVERAGE_TITLE, - search_title=C.SEARCH_TITLE + menu_itms=menu_itms ) diff --git a/csit.infra.dash/app/cdash/stats/layout.py b/csit.infra.dash/app/cdash/stats/layout.py index 655c61c078..616a4028e6 100644 --- a/csit.infra.dash/app/cdash/stats/layout.py +++ b/csit.infra.dash/app/cdash/stats/layout.py @@ -196,7 +196,7 @@ class Layout: "dd-tbeds-value": self._default["tbed"], "al-job-children": html.A( self._default["job"], - href=f"{C.URL_JENKINS}{self._default['job']}", + href=f"{C.URL_CICD}{self._default['job']}", target="_blank" ) } @@ -631,7 +631,7 @@ class Layout: "al-job-children": html.A( self._default["job"], href=( - f"{C.URL_JENKINS}" + f"{C.URL_CICD}" f"{self._default['job']}" ), target="_blank" @@ -654,7 +654,7 @@ class Layout: { "al-job-children": html.A( job, - href=f"{C.URL_JENKINS}{job}", + href=f"{C.URL_CICD}{job}", target="_blank" ) } diff --git a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 index 7b0dadc5a0..97a71e358f 100644 --- a/csit.infra.dash/app/cdash/templates/base_layout.jinja2 +++ b/csit.infra.dash/app/cdash/templates/base_layout.jinja2 @@ -6,7 +6,7 @@ <header class="mb-auto"> <div> <h3 class="float-md-start mb-0 text-white"> - CSIT-Dash + {{ brand }} </h3> </div> </header> @@ -20,52 +20,18 @@ {{ description }} </p> <p class="lead"> - <p> - <a href="/trending/" class="btn btn-primary fw-bold w-25"> - {{ trending_title }} - </a> - </p> - <p> - <a href="/report/" class="btn btn-primary fw-bold w-25"> - {{ report_title }} - </a> - </p> - <p> - <a href="/comparisons/" class="btn btn-primary fw-bold w-25"> - {{ comp_title }} - </a> - </p> - <p> - <p> - <a href="/coverage/" class="btn btn-primary fw-bold w-25"> - {{ cov_title }} - </a> - </p> - <p> - <a href="/stats/" class="btn btn-primary fw-bold w-25"> - {{ stats_title }} - </a> - </p> - <p> - <a href="/news/" class="btn btn-primary fw-bold w-25"> - {{ news_title }} - </a> - </p> - <p> - <a href="/search/" class="btn btn-primary fw-bold w-25"> - {{ search_title }} - </a> - </p> - <p> - <a href="/cdocs/" class="btn btn-primary fw-bold w-25"> - Documentation - </a> - </p> + {% for itm in menu_itms %} + <p> + <a href="{{ itm['path'] }}" class="btn btn-primary fw-bold w-25"> + {{ itm['title'] }} + </a> + </p> + {% endfor %} </p> </main> <footer class="mt-auto text-white-50"> - <p>Copyright © 2016-2023 <a href="https://fd.io" class="text-white">The Fast Data Project</a>, a series of LF Projects, LLC.</p> + <p>Copyright © 2016-2024 <a href="https://fd.io" class="text-white">The Fast Data Project</a>, a series of LF Projects, LLC.</p> </footer> </div> {% endblock %} diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py index 3db0d50f4f..bafa7b7f42 100644 --- a/csit.infra.dash/app/cdash/utils/constants.py +++ b/csit.infra.dash/app/cdash/utils/constants.py @@ -17,11 +17,75 @@ does not need to be hard coded here, but can be read from environment variables. """ +import os import logging from dash import html +def get_str_from_env(env_var_name: str, default_value: str) -> str: + """Attempt to read string from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + If environment variable exists, but is empty (and default is not), + empty string is returned. + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if the env var does not exist. + :type env_var_names: str + :type default_value: str + :returns: The value read, or default value. + :rtype: str + """ + prefix = "CSIT_" + env_str = os.environ.get(prefix + env_var_name, None) + if env_str is not None: + return env_str + return default_value + + +def get_int_from_env(env_var_name: str, default_value: int) -> int: + """Attempt to read int from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + String value is read, default is returned also if conversion fails. + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if read or conversion fails. + :type env_var_names: str + :type default_value: int + :returns: The value read, or default value. + :rtype: int + """ + try: + return int(get_str_from_env(env_var_name, str())) + except ValueError: + return default_value + + +def get_bool_from_env(env_var_name: str, default_value: bool) -> bool: + """Attempt to read bool from environment variable, return that or default. + + The environment variable must start with perfix "CSIT_". + + :param env_var_name: Base name of environment variable to attempt to read. + :param default_value: Value to return if read or conversion fails. + :type env_var_names: str + :type default_value: bool + :returns: The value read, or default value. + :rtype: bool + """ + env_str = get_str_from_env(env_var_name, str()).lower() + if env_str in ("true", "yes", "y", "1"): + return True + elif env_str in ("false", "no", "n", "0"): + return False + else: + return default_value + + class Constants: """Constants used in CDash. """ @@ -29,14 +93,24 @@ class Constants: ############################################################################ # General, application wide constants. + # Select applications to start. + START_TRENDING = get_bool_from_env("START_TRENDING", True) + START_REPORT = get_bool_from_env("START_REPORT", True) + START_COMPARISONS = get_bool_from_env("START_COMPARISONS", True) + START_COVERAGE = get_bool_from_env("START_COVERAGE", True) + START_STATISTICS = get_bool_from_env("START_STATISTICS", True) + START_FAILURES = get_bool_from_env("START_FAILURES", True) + START_SEARCH = get_bool_from_env("START_SEARCH", True) + START_DOC = get_bool_from_env("START_DOC", True) + # Logging settings. LOG_LEVEL = logging.INFO LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s" LOG_DATE_FORMAT = "%Y/%m/%d %H:%M:%S" # The application title. - TITLE = "FD.io CSIT" - BRAND = "CSIT-Dash" + TITLE = get_str_from_env("TITLE", "FD.io CSIT") + BRAND = get_str_from_env("BRAND", "CSIT-Dash") # The application description. DESCRIPTION = "Performance Dashboard" @@ -45,14 +119,17 @@ class Constants: EXTERNAL_STYLESHEETS = ["/static/dist/css/bootstrap.css", ] # URL to Jenkins - URL_JENKINS = "https://jenkins.fd.io/job/" + URL_CICD = get_str_from_env("URL_CICD", "https://jenkins.fd.io/job/") # URL to logs - URL_LOGS = "https://logs.fd.io/vex-yul-rot-jenkins-1/" + URL_LOGS = get_str_from_env( + "URL_LOGS", "https://logs.fd.io/vex-yul-rot-jenkins-1/" + ) # URL to the documentation - URL_DOC_TRENDING = "https://csit.fd.io/cdocs/methodology/trending/analysis/" - URL_DOC_REL_NOTES = "https://csit.fd.io/cdocs/release_notes/current/" + URL_DOC = get_str_from_env("URL_DOC", "https://csit.fd.io/cdocs/") + URL_DOC_TRENDING = URL_DOC + "methodology/trending/analysis/" + URL_DOC_REL_NOTES = URL_DOC + "release_notes/current/" # Path and name of the file specifying the HTML layout of the dash # application. @@ -82,7 +159,7 @@ class Constants: # now back to the past. # TIME_PERIOD = None - means all data (max MAX_TIME_PERIOD days) is read. # TIME_PERIOD = MAX_TIME_PERIOD - is the default value - TIME_PERIOD = MAX_TIME_PERIOD # [days] + TIME_PERIOD = get_int_from_env("TIME_PERIOD", MAX_TIME_PERIOD) # [days] ############################################################################ # General, application wide, layout affecting constants. @@ -467,3 +544,9 @@ class Constants: SEARCH_DOWNLOAD_FILE_NAME = "search_data.csv" ############################################################################ + # Documentation. + + # The title. + DOC_TITLE = "Documentation" + + ############################################################################ diff --git a/csit.infra.dash/app/cdash/utils/utils.py b/csit.infra.dash/app/cdash/utils/utils.py index 306b4f60d1..e203dfbccd 100644 --- a/csit.infra.dash/app/cdash/utils/utils.py +++ b/csit.infra.dash/app/cdash/utils/utils.py @@ -480,37 +480,42 @@ def navbar_trending(active: tuple): :returns: Navigation bar. :rtype: dbc.NavbarSimple """ + children = list() + if C.START_TRENDING: + children.append(dbc.NavItem(dbc.NavLink( + C.TREND_TITLE, + active=active[0], + external_link=True, + href="/trending" + ))) + if C.START_FAILURES: + children.append(dbc.NavItem(dbc.NavLink( + C.NEWS_TITLE, + active=active[1], + external_link=True, + href="/news" + ))) + if C.START_STATISTICS: + children.append(dbc.NavItem(dbc.NavLink( + C.STATS_TITLE, + active=active[2], + external_link=True, + href="/stats" + ))) + if C.START_SEARCH: + children.append(dbc.NavItem(dbc.NavLink( + C.SEARCH_TITLE, + active=active[3], + external_link=True, + href="/search" + ))) + if C.START_DOC: + children.append(dbc.NavItem(dbc.NavLink( + "Documentation", + id="btn-documentation", + ))) return dbc.NavbarSimple( - children=[ - dbc.NavItem(dbc.NavLink( - C.TREND_TITLE, - active=active[0], - external_link=True, - href="/trending" - )), - dbc.NavItem(dbc.NavLink( - C.NEWS_TITLE, - active=active[1], - external_link=True, - href="/news" - )), - dbc.NavItem(dbc.NavLink( - C.STATS_TITLE, - active=active[2], - external_link=True, - href="/stats" - )), - dbc.NavItem(dbc.NavLink( - C.SEARCH_TITLE, - active=active[3], - external_link=True, - href="/search" - )), - dbc.NavItem(dbc.NavLink( - "Documentation", - id="btn-documentation", - )) - ], + children=children, id="navbarsimple-main", brand=C.BRAND, brand_href="/", @@ -529,38 +534,43 @@ def navbar_report(active: tuple): :returns: Navigation bar. :rtype: dbc.NavbarSimple """ + children = list() + if C.START_REPORT: + children.append(dbc.NavItem(dbc.NavLink( + C.REPORT_TITLE, + active=active[0], + external_link=True, + href="/report" + ))) + if C.START_COMPARISONS: + children.append(dbc.NavItem(dbc.NavLink( + "Comparisons", + active=active[1], + external_link=True, + href="/comparisons" + ))) + if C.START_COVERAGE: + children.append(dbc.NavItem(dbc.NavLink( + "Coverage Data", + active=active[2], + external_link=True, + href="/coverage" + ))) + if C.START_SEARCH: + children.append(dbc.NavItem(dbc.NavLink( + C.SEARCH_TITLE, + active=active[3], + external_link=True, + href="/search" + ))) + if C.START_DOC: + children.append(dbc.NavItem(dbc.NavLink( + "Documentation", + id="btn-documentation", + ))) return dbc.NavbarSimple( + children=children, id="navbarsimple-main", - children=[ - dbc.NavItem(dbc.NavLink( - C.REPORT_TITLE, - active=active[0], - external_link=True, - href="/report" - )), - dbc.NavItem(dbc.NavLink( - "Comparisons", - active=active[1], - external_link=True, - href="/comparisons" - )), - dbc.NavItem(dbc.NavLink( - "Coverage Data", - active=active[2], - external_link=True, - href="/coverage" - )), - dbc.NavItem(dbc.NavLink( - C.SEARCH_TITLE, - active=active[3], - external_link=True, - href="/search" - )), - dbc.NavItem(dbc.NavLink( - "Documentation", - id="btn-documentation", - )) - ], brand=C.BRAND, brand_href="/", brand_external_link=True, diff --git a/csit.infra.dash/docker-compose.yaml b/csit.infra.dash/docker-compose.yaml index 5ed71345fb..a3b5e1a683 100644 --- a/csit.infra.dash/docker-compose.yaml +++ b/csit.infra.dash/docker-compose.yaml @@ -6,6 +6,20 @@ services: environment: FLASK_DEBUG: 1 FLASK_ENV: "development" + CSIT_START_TRENDING: "True" + CSIT_START_REPORT: "True" + CSIT_START_COMPARISONS: "True" + CSIT_START_COVERAGE: "True" + CSIT_START_STATISTICS: "True" + CSIT_START_FAILURES: "True" + CSIT_START_SEARCH: "True" + CSIT_START_DOC: "True" + CSIT_TITLE: "FD.io CSIT" + CSIT_BRAND: "CSIT-Dash" + CSIT_URL_CICD: "https://jenkins.fd.io/job/" + CSIT_URL_LOGS: "https://logs.fd.io/vex-yul-rot-jenkins-1/" + CSIT_URL_DOC: "https://csit.fd.io/cdocs/" + CSIT_TIME_PERIOD: 250 mem_limit: "16g" ports: - "5000:5000" |