From 196ed0599148d6c8ed0e2ca82ad03e129e26428a Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Fri, 28 Jul 2017 13:06:56 +0200 Subject: CSIT-618 Add LaTeX support to release report - Add option to generate PDF from release report. - Conditional content - Static plot conversion Change-Id: I7c4f0b41898791d1cab446d0fc3b07220380e1b8 Signed-off-by: Peter Mikus --- resources/tools/report_gen/conf.py | 87 +++++++++++++++++++++- resources/tools/report_gen/fdio.svg | 25 +++++++ resources/tools/report_gen/requirements.txt | 2 +- resources/tools/report_gen/run_report.cfg | 3 +- resources/tools/report_gen/run_report.sh | 108 ++++++++++++++++++++-------- 5 files changed, 193 insertions(+), 32 deletions(-) create mode 100644 resources/tools/report_gen/fdio.svg (limited to 'resources/tools/report_gen') diff --git a/resources/tools/report_gen/conf.py b/resources/tools/report_gen/conf.py index 89dd45824f..87120a5baf 100644 --- a/resources/tools/report_gen/conf.py +++ b/resources/tools/report_gen/conf.py @@ -30,7 +30,8 @@ sys.path.insert(0, os.path.abspath('.')) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxcontrib.programoutput'] +extensions = ['sphinxcontrib.programoutput', + 'sphinx.ext.ifconfig'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -109,7 +110,6 @@ todo_include_todos = False # pdf_documents = [('index', u'rst2pdf', u'Sample rst2pdf doc', u'Your Name'),] - # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -135,3 +135,86 @@ html_context = { '_static/theme_overrides.css', # overrides for wide tables in RTD theme ], } + +# -- Options for LaTeX output --------------------------------------------- + +latex_engine = 'pdflatex' + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + #'papersize': 'a4paper', + + # The font size ('10pt', '11pt' or '12pt'). + # + #'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + 'preamble': r''' + \usepackage{pdfpages} + \usepackage{svg} + \usepackage{charter} + \usepackage[defaultsans]{lato} + \usepackage{inconsolata} + ''', + + # Latex figure (float) alignment + # + 'figure_align': 'H', + + # Latex other setup + # + 'extraclassoptions': 'openany', + 'sphinxsetup': r''' + TitleColor={RGB}{225,38,40}, + InnerLinkColor={RGB}{62,62,63}, + OuterLinkColor={RGB}{225,38,40}, + shadowsep=0pt, + shadowsize=0pt, + shadowrule=0pt + ''', + 'fontpkg': r''' + \renewcommand{\familydefault}{\sfdefault} + ''' +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'csit.tex', u'CSIT Report', + u'', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +latex_logo = 'fdio.pdf' + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +latex_use_parts = True + +# If true, show page references after internal links. +# +latex_show_pagerefs = True + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True diff --git a/resources/tools/report_gen/fdio.svg b/resources/tools/report_gen/fdio.svg new file mode 100644 index 0000000000..f0dcc87cd4 --- /dev/null +++ b/resources/tools/report_gen/fdio.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + diff --git a/resources/tools/report_gen/requirements.txt b/resources/tools/report_gen/requirements.txt index 66a1b2b531..fc0ff8c728 100644 --- a/resources/tools/report_gen/requirements.txt +++ b/resources/tools/report_gen/requirements.txt @@ -1,6 +1,6 @@ Sphinx sphinx-rtd-theme -plotly==2.0.12 +plotly lxml==3.5.0 robotframework==2.9.2 sphinxcontrib-programoutput diff --git a/resources/tools/report_gen/run_report.cfg b/resources/tools/report_gen/run_report.cfg index 747beda725..3ff9e605a5 100644 --- a/resources/tools/report_gen/run_report.cfg +++ b/resources/tools/report_gen/run_report.cfg @@ -10,6 +10,7 @@ typeset -A JOB DIR[WORKING]=_tmp DIR[BUILD]=_build +DIR[BUILD,LATEX]=_build_latex DIR[RST]=../../../docs/report DIR[STATIC]=${DIR[BUILD]}/_static @@ -44,7 +45,7 @@ URL[JENKINS,HC]='https://jenkins.fd.io/view/hc2vpp/job' JOB[PERF,VPP]=csit-vpp-perf-1707-all JOB[PERF,VPP,BLD]="9 10 13 14 15 16 17 18 19 21" -JOB[PERF,VPP,FBLD]=10 +JOB[PERF,VPP,FBLD]=22 JOB[PERF,DPDK]=csit-dpdk-perf-1707-all JOB[PERF,DPDK,BLD]="1 2 3 4 5 6 7 8 9 10" JOB[PERF,DPDK,FBLD]=10 diff --git a/resources/tools/report_gen/run_report.sh b/resources/tools/report_gen/run_report.sh index a40d91dac7..bf6d5844dd 100755 --- a/resources/tools/report_gen/run_report.sh +++ b/resources/tools/report_gen/run_report.sh @@ -2,12 +2,32 @@ set -x -# Build locally without jenkins integrations +# Process parameters +# Debugging DEBUG=0 +# Latex Builder +LATEX=0 + +for i in "$@"; do +case $i in + --debug) + DEBUG=1 + ;; + --latex) + LATEX=1 + ;; + *) + # unknown option + ;; +esac +done # Script directory SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# Get actual date +DATE=$(date -u '+%d-%b-%Y') + # Load configuration source ${SCRIPT_DIR}/run_report.cfg @@ -15,6 +35,12 @@ source ${SCRIPT_DIR}/run_report.cfg sudo apt-get -y update sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \ zlib1g-dev unzip +if [[ ${LATEX} -eq 1 ]] ; +then + sudo apt-get -y install librsvg2-bin texlive-latex-recommended \ + texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra latexmk wkhtmltopdf + sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' /usr/share/texlive/texmf-dist/web2c/texmf.cnf +fi # Clean-up when finished trap 'rm -rf ${DIR[WORKING]}; exit' EXIT @@ -22,6 +48,7 @@ trap 'rm -rf ${DIR[WORKING]}; exit' ERR # Remove the old build rm -rf ${DIR[BUILD]} || true +rm -rf ${DIR[BUILD,LATEX]} || true rm -rf ${DIR[WORKING]} || true # Create working directories @@ -55,14 +82,17 @@ blds=${JOB[PERF,VPP,BLD]} for i in ${blds[@]}; do curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/output_perf_data.xml \ --output ${DIR[PLOT,VPP]}/${JOB[PERF,VPP]}-${i}.xml - #curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/output_perf_data.json \ - # --output ${DIR[PLOT,VPP]}/${JOB[PERF,VPP]}-${i}.json if [[ ${DEBUG} -eq 0 ]] ; then curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/\*zip\*/robot-plugin.zip \ --output ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${i}.zip fi done +if [[ ${DEBUG} -eq 0 ]] ; +then + curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${JOB[PERF,VPP,FBLD]}/robot/report/\*zip\*/robot-plugin.zip \ + --output ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${JOB[PERF,VPP,FBLD]}.zip +fi # Archive trending cp ${DIR[PLOT,VPP]}/* ${DIR[STATIC,TREND]} blds=${JOB[1704,VPP,BLD]} @@ -81,8 +111,6 @@ blds=${JOB[PERF,DPDK,BLD]} for i in ${blds[@]}; do curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/output_perf_data.xml \ --output ${DIR[PLOT,DPDK]}/${JOB[PERF,DPDK]}-${i}.xml - #curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/output_perf_data.json \ - # --output ${DIR[PLOT,DPDK]}/${JOB[PERF,DPDK]}-${i}.json if [[ ${DEBUG} -eq 0 ]] ; then curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/\*zip\*/robot-plugin.zip \ @@ -206,29 +234,6 @@ fi # Delete temporary json files find ${DIR[RST]} -name "*.json" -type f -delete -# Generate the documentation -DATE=$(date -u '+%d-%b-%Y') -sphinx-build -v -c . -a -b html -E \ - -D release=$1 -D version="$1 report - $DATE" \ - ${DIR[RST]} ${DIR[BUILD]}/ - -# Patch the CSS for tables layout -cat - > ${DIR[CSS_PATCH_FILE]} <<"_EOF" -/* override table width restrictions */ -@media screen and (min-width: 767px) { - .wy-table-responsive table td, .wy-table-responsive table th { - white-space: normal !important; - } - - .wy-table-responsive { - font-size: small; - margin-bottom: 24px; - max-width: 100%; - overflow: visible !important; - } -} -_EOF - # Plot packets per second # VPP L2 sel1 @@ -237,6 +242,7 @@ python run_plot.py --input ${DIR[PLOT,VPP]} \ --output ${DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc \ --title "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" \ --xpath '//*[@framesize="64B" and contains(@tags,"BASE") and contains(@tags,"NDRDISC") and contains(@tags,"1T1C") and (contains(@tags,"L2BDMACSTAT") or contains(@tags,"L2BDMACLRN") or contains(@tags,"L2XCFWD")) and not(contains(@tags,"VHOST"))]' + python run_plot.py --input ${DIR[PLOT,VPP]} \ --output ${DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc \ --title "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" \ @@ -571,6 +577,52 @@ python run_plot.py --input ${DIR[PLOT,DPDK]} \ --title "64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" \ --xpath '//*[@framesize="64B" and contains(@tags,"BASE") and contains(@tags,"NDRDISC") and contains(@tags,"2T2C") and contains(@tags,"IP4FWD")]' --latency lat_50 +# HTML BUILDER +sphinx-build -v -c . -a -b html -E \ + -D release=$1 -D version="$1 report - $DATE" \ + ${DIR[RST]} ${DIR[BUILD]}/ + +# Patch the CSS for tables layout +cat - > ${DIR[CSS_PATCH_FILE]} <<"_EOF" +/* override table width restrictions */ +@media screen and (min-width: 767px) { + .wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal !important; + } + + .wy-table-responsive { + font-size: small; + margin-bottom: 24px; + max-width: 100%; + overflow: visible !important; + } +} +_EOF + +# LATEX BUILDER +if [[ ${LATEX} -eq 1 ]] ; +then + # Convert PyPLOT graphs in HTML format to PDF. + for f in ${DIR[STATIC,VPP]}/*; do + wkhtmltopdf ${f} ${f%.html}.pdf + done + for f in ${DIR[STATIC,DPDK]}/*; do + wkhtmltopdf ${f} ${f%.html}.pdf + done + rsvg-convert -z 10 -f pdf -o fdio.pdf fdio.svg + + # Generate the LaTeX documentation + sphinx-build -v -c . -a -b latex -E \ + -D release=$1 -D version="$1 report - $DATE" \ + ${DIR[RST]} ${DIR[BUILD,LATEX]} + cd ${DIR[BUILD,LATEX]} + pdflatex -shell-escape -interaction nonstopmode csit.tex || true + pdflatex -interaction nonstopmode csit.tex || true + cp csit.pdf ../${DIR[STATIC,ARCH]}/csit_$1.pdf + cd ${SCRIPT_DIR} + rm -f fdio.pdf +fi + # Create archive echo Creating csit.report.tar.gz ... tar -czvf ./csit.report.tar.gz ${DIR[BUILD]} -- cgit 1.2.3-korg