CSIT-618 Add LaTeX support to release report

- Add option to generate PDF from release report.
- Conditional content
- Static plot conversion

Change-Id: I7c4f0b41898791d1cab446d0fc3b07220380e1b8
Signed-off-by: Peter Mikus <pmikus@cisco.com>

5 files changed, 193 insertions, 32 deletions

diff --git a/resources/tools/report_gen/conf.py b/resources/tools/report_gen/conf.py
index 89dd45824f..87120a5baf 100644
--- a/resources/tools/report_gen/conf.py
+++ b/resources/tools/report_gen/conf.py
@@ -30,7 +30,8 @@ sys.path.insert(0, os.path.abspath('.'))
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinxcontrib.programoutput']
+extensions = ['sphinxcontrib.programoutput',
+              'sphinx.ext.ifconfig']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -109,7 +110,6 @@ todo_include_todos = False
 
 # pdf_documents = [('index', u'rst2pdf', u'Sample rst2pdf doc', u'Your Name'),]
 
-
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -135,3 +135,86 @@ html_context = {
         '_static/theme_overrides.css',  # overrides for wide tables in RTD theme
         ],
     }
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_engine = 'pdflatex'
+
+latex_elements = {
+     # The paper size ('letterpaper' or 'a4paper').
+     #
+     #'papersize': 'a4paper',
+
+     # The font size ('10pt', '11pt' or '12pt').
+     #
+     #'pointsize': '10pt',
+
+     # Additional stuff for the LaTeX preamble.
+     #
+     'preamble': r'''
+         \usepackage{pdfpages}
+         \usepackage{svg}
+         \usepackage{charter}
+         \usepackage[defaultsans]{lato}
+         \usepackage{inconsolata}
+     ''',
+
+     # Latex figure (float) alignment
+     #
+     'figure_align': 'H',
+
+     # Latex other setup
+     #
+     'extraclassoptions': 'openany',
+     'sphinxsetup': r'''
+         TitleColor={RGB}{225,38,40},
+         InnerLinkColor={RGB}{62,62,63},
+         OuterLinkColor={RGB}{225,38,40},
+         shadowsep=0pt,
+         shadowsize=0pt,
+         shadowrule=0pt
+     ''',
+     'fontpkg': r'''
+         \renewcommand{\familydefault}{\sfdefault}
+     '''
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'csit.tex', u'CSIT Report',
+     u'', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+latex_logo = 'fdio.pdf'
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+latex_use_parts = True
+
+# If true, show page references after internal links.
+#
+latex_show_pagerefs = True
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# It false, will not define \strong, \code, 	itleref, \crossref ... but only
+# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
+# packages.
+#
+# latex_keep_old_macro_names = True
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
diff --git a/resources/tools/report_gen/fdio.svg b/resources/tools/report_gen/fdio.svg
new file mode 100644
index 0000000000..f0dcc87cd4
--- /dev/null
+++ b/resources/tools/report_gen/fdio.svg
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>

+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">

+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="630px" height="394px" version="1.1" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"

+viewBox="0 0 94500000 59062500"

+ xmlns:xlink="http://www.w3.org/1999/xlink">

+ <defs>

+  <style type="text/css">

+   <![CDATA[

+    .fil2 {fill:#3C4641}

+    .fil1 {fill:#3E4742}

+    .fil0 {fill:#ED3341}

+   ]]>

+  </style>

+ </defs>

+ <g id="Layer_x0020_1">

+  <path class="fil0" d="M42892251 15970973c234833,2555989 1402971,3784607 3633525,3195163 64614,-17483 464586,-605509 524121,-613778 100406,-13939 -325789,592397 -234951,589562 663981,-21026 1166839,235659 2703054,-1575788 -125567,2507794 385088,1931698 753047,3733577 -1039736,2409632 -1745415,307125 -1486485,4471622 -4668654,-2126 -6967721,-648979 -9839813,2159679 -1571417,1536924 -3134683,5759184 -3805515,8365022 -725760,124740 -1467939,37800 -1887401,510536 -484194,545619 -431747,1408050 35674,1833418 379890,345516 987289,308424 1519678,355793 800769,71348 1142151,473327 1136244,1236651 -6615,848964 -451001,1192826 -1631306,1191054 -3702746,-5316 -8320961,16538 -11713866,45596 -1026624,8741 -1698992,464822 -1746714,1344971 -50676,931416 521640,1260039 1499833,1295831 4286993,156988 8304069,-117653 12887201,141041 783996,103714 1189755,583183 1168256,1106004 -35201,855461 -707923,1099626 -1411594,1145104 -5730598,370440 -13969581,-91665 -19611231,141278 -797344,32839 -1694621,179786 -1702890,1222476 -6615,840459 589798,1177943 1399899,1281538 2377738,7088 5327556,1418 7519956,1890 2596151,591 5192775,4134 7788926,-118l-1007843 3222096 9601909 24334 3086606 -8484328c2943911,-111628 4325029,-862076 6023666,-2197361 944528,-742534 2909773,-3271590 3377903,-4985466l-6699578 -78081 1196016 -3619114c2873627,-19727 7980761,562984 10127447,-748913 1534089,-937440 4106970,-3980458 4422954,-6498411 -1731358,-38863 -8963443,287753 -9479649,-290469 -811991,-1588073 1028633,-2402426 1978121,-3132911 1388678,-1068441 1793846,-1560904 1785578,-3760746 230108,313386 87413,679809 294368,826048 502740,355320 1446913,224319 3038411,773010 1055447,363943 2115855,607399 3274661,404696 606218,-105958 570662,-121196 999692,-407413 -87767,-924683 -247472,-1025089 -709813,-1469003 1446086,654176 966617,1888937 966499,3676168 0,1623983 504512,3115429 -2599,4527259 -1397891,3892219 -3372351,4768234 -4125752,6086745l4834856 25043 -4320540 12061626c-2523504,20318 -10295893,-341499 -12094937,331459 -2551264,954568 -5105126,4809341 -5427844,6962878 3747398,-8151 17429226,243219 20078179,-244755 5769579,-1062534 6837902,-6323231 8783066,-11759344 1441716,-4029244 5291646,-11853371 -1699464,-14191183 -1193535,-382253 -1853145,-295667 -2077228,-395955 -216523,-96863 4489,-368078 -1209600,-1404861 -1196488,-1140379 -417690,-2367934 118716,-3662820 1265001,-3005691 3370461,-2733058 3370343,-6944096 -236,-4388934 2680611,-3141771 3022583,-4763627 307479,-1458135 -1345326,-645317 -2307336,-1215388 -794273,-470610 -170809,-694693 -1801524,-842940l-178251 -1084151c-406114,246999 -713003,925509 -824631,912870 -159705,-18191 -35083,-801478 -213216,-759780 -33075,7796 -73828,19845 -123323,42643 -162068,75009 -368668,285036 -609053,585900 -1161759,1454355 -3112358,5035433 -4642076,5789779 -3004746,1481878 -5309719,91074 -10962709,3162206 -2587056,1405451 -4703383,1914216 -7362259,348233z"/>

+  <path class="fil1" d="M81444116 48006591c-917831,-4596598 5486670,-5710989 6489669,-1741753 1221649,4834502 -5678387,5805017 -6489669,1741753zm1873463 -5811986c-6608858,1801052 -4202297,11851718 2867130,10021253 6433324,-1665681 4111577,-11922947 -2867130,-10021253z"/>

+  <polygon class="fil2" points="76840667,42255321 75263698,42301508 75172978,52072571 76944144,52066547 "/>

+  <path class="fil1" d="M70635206 52314137c1084388,307834 2141370,-714302 1520859,-1977413 -474863,-966853 -2784797,-983627 -2488776,930589 89421,578931 388041,882158 967916,1046824z"/>

+  <path class="fil1" d="M76175978 40542272c695402,-45006 1114037,-372448 1062889,-1137426 -18309,-275349 -120251,-539595 -294013,-710876 -609761,-601611 -1840151,-411548 -1952016,509001 -111628,919485 435173,1387733 1183140,1339301z"/>

+  <path class="fil0" d="M14086524 36346354l15995897 0c719972,0 1308943,589089 1308943,1308943l0 0c0,719854 -589089,1308943 -1308943,1308943l-15995897 0c-719736,0 -1308943,-588971 -1308943,-1308943l0 0c0,-719972 589089,-1308943 1308943,-1308943z"/>

+  <path class="fil0" d="M20652975 31242173l13718919 0c719972,0 1308943,589326 1308943,1308943l0 118c0,719618 -589208,1308943 -1308943,1308943l-13718919 0c-719736,0 -1308943,-588971 -1308943,-1308943l0 -118c0,-719972 588971,-1308943 1308943,-1308943z"/>

+  <path class="fil0" d="M6885624 46609763l3154410 0c719972,0 1308943,589208 1308943,1308943l0 0c0,719736 -589326,1308943 -1308943,1308943l-3154410 0c-719736,0 -1308943,-588971 -1308943,-1308943l0 0c0,-719972 588971,-1308943 1308943,-1308943z"/>

+ </g>

+</svg>

diff --git a/resources/tools/report_gen/requirements.txt b/resources/tools/report_gen/requirements.txt
index 66a1b2b531..fc0ff8c728 100644
--- a/resources/tools/report_gen/requirements.txt
+++ b/resources/tools/report_gen/requirements.txt
@@ -1,6 +1,6 @@
 Sphinx
 sphinx-rtd-theme
-plotly==2.0.12
+plotly
 lxml==3.5.0
 robotframework==2.9.2
 sphinxcontrib-programoutput
diff --git a/resources/tools/report_gen/run_report.cfg b/resources/tools/report_gen/run_report.cfg
index 747beda725..3ff9e605a5 100644
--- a/resources/tools/report_gen/run_report.cfg
+++ b/resources/tools/report_gen/run_report.cfg
@@ -10,6 +10,7 @@ typeset -A JOB
 
 DIR[WORKING]=_tmp
 DIR[BUILD]=_build
+DIR[BUILD,LATEX]=_build_latex
 DIR[RST]=../../../docs/report
 
 DIR[STATIC]=${DIR[BUILD]}/_static
@@ -44,7 +45,7 @@ URL[JENKINS,HC]='https://jenkins.fd.io/view/hc2vpp/job'
 
 JOB[PERF,VPP]=csit-vpp-perf-1707-all
 JOB[PERF,VPP,BLD]="9 10 13 14 15 16 17 18 19 21"
-JOB[PERF,VPP,FBLD]=10
+JOB[PERF,VPP,FBLD]=22
 JOB[PERF,DPDK]=csit-dpdk-perf-1707-all
 JOB[PERF,DPDK,BLD]="1 2 3 4 5 6 7 8 9 10"
 JOB[PERF,DPDK,FBLD]=10
diff --git a/resources/tools/report_gen/run_report.sh b/resources/tools/report_gen/run_report.sh
index a40d91dac7..bf6d5844dd 100755
--- a/resources/tools/report_gen/run_report.sh
+++ b/resources/tools/report_gen/run_report.sh
@@ -2,12 +2,32 @@
 
 set -x
 
-# Build locally without jenkins integrations
+# Process parameters
+# Debugging
 DEBUG=0
+# Latex Builder
+LATEX=0
+
+for i in "$@"; do
+case $i in
+    --debug)
+        DEBUG=1
+    ;;
+    --latex)
+        LATEX=1
+    ;;
+    *)
+        # unknown option
+    ;;
+esac
+done
 
 # Script directory
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+# Get actual date
+DATE=$(date -u '+%d-%b-%Y')
+
 # Load configuration
 source ${SCRIPT_DIR}/run_report.cfg
 
@@ -15,6 +35,12 @@ source ${SCRIPT_DIR}/run_report.cfg
 sudo apt-get -y update
 sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \
     zlib1g-dev unzip
+if [[ ${LATEX} -eq 1 ]] ;
+then
+    sudo apt-get -y install librsvg2-bin texlive-latex-recommended \
+        texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra latexmk wkhtmltopdf
+    sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' /usr/share/texlive/texmf-dist/web2c/texmf.cnf
+fi
 
 # Clean-up when finished
 trap 'rm -rf ${DIR[WORKING]}; exit' EXIT
@@ -22,6 +48,7 @@ trap 'rm -rf ${DIR[WORKING]}; exit' ERR
 
 # Remove the old build
 rm -rf ${DIR[BUILD]} || true
+rm -rf ${DIR[BUILD,LATEX]} || true
 rm -rf ${DIR[WORKING]} || true
 
 # Create working directories
@@ -55,14 +82,17 @@ blds=${JOB[PERF,VPP,BLD]}
 for i in ${blds[@]}; do
     curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/output_perf_data.xml \
         --output ${DIR[PLOT,VPP]}/${JOB[PERF,VPP]}-${i}.xml
-    #curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/output_perf_data.json \
-    #    --output ${DIR[PLOT,VPP]}/${JOB[PERF,VPP]}-${i}.json
     if [[ ${DEBUG} -eq 0 ]] ;
     then
         curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${i}/robot/report/\*zip\*/robot-plugin.zip \
             --output ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${i}.zip
     fi
 done
+if [[ ${DEBUG} -eq 0 ]] ;
+then
+    curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,VPP]}/${JOB[PERF,VPP,FBLD]}/robot/report/\*zip\*/robot-plugin.zip \
+        --output ${DIR[STATIC,ARCH]}/${JOB[PERF,VPP]}-${JOB[PERF,VPP,FBLD]}.zip
+fi
 # Archive trending
 cp ${DIR[PLOT,VPP]}/* ${DIR[STATIC,TREND]}
 blds=${JOB[1704,VPP,BLD]}
@@ -81,8 +111,6 @@ blds=${JOB[PERF,DPDK,BLD]}
 for i in ${blds[@]}; do
     curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/output_perf_data.xml \
         --output ${DIR[PLOT,DPDK]}/${JOB[PERF,DPDK]}-${i}.xml
-    #curl --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/output_perf_data.json \
-    #    --output ${DIR[PLOT,DPDK]}/${JOB[PERF,DPDK]}-${i}.json
     if [[ ${DEBUG} -eq 0 ]] ;
     then
         curl --fail --silent ${URL[JENKINS,CSIT]}/${JOB[PERF,DPDK]}/${i}/robot/report/\*zip\*/robot-plugin.zip \
@@ -206,29 +234,6 @@ fi
 # Delete temporary json files
 find ${DIR[RST]} -name "*.json" -type f -delete
 
-# Generate the documentation
-DATE=$(date -u '+%d-%b-%Y')
-sphinx-build -v -c . -a -b html -E \
-    -D release=$1 -D version="$1 report - $DATE" \
-    ${DIR[RST]} ${DIR[BUILD]}/
-
-# Patch the CSS for tables layout
-cat - > ${DIR[CSS_PATCH_FILE]} <<"_EOF"
-/* override table width restrictions */
-@media screen and (min-width: 767px) {
-    .wy-table-responsive table td, .wy-table-responsive table th {
-        white-space: normal !important;
-    }
-
-    .wy-table-responsive {
-        font-size: small;
-        margin-bottom: 24px;
-        max-width: 100%;
-        overflow: visible !important;
-    }
-}
-_EOF
-
 # Plot packets per second
 
 # VPP L2 sel1
@@ -237,6 +242,7 @@ python run_plot.py --input ${DIR[PLOT,VPP]} \
     --output ${DIR[STATIC,VPP]}/64B-1t1c-l2-sel1-ndrdisc \
     --title "64B-1t1c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" \
     --xpath '//*[@framesize="64B" and contains(@tags,"BASE") and contains(@tags,"NDRDISC") and contains(@tags,"1T1C") and (contains(@tags,"L2BDMACSTAT") or contains(@tags,"L2BDMACLRN") or contains(@tags,"L2XCFWD")) and not(contains(@tags,"VHOST"))]'
+
 python run_plot.py --input ${DIR[PLOT,VPP]} \
     --output ${DIR[STATIC,VPP]}/64B-2t2c-l2-sel1-ndrdisc \
     --title "64B-2t2c-(eth|dot1q|dot1ad)-(l2xcbase|l2bdbasemaclrn)-ndrdisc" \
@@ -571,6 +577,52 @@ python run_plot.py --input ${DIR[PLOT,DPDK]} \
     --title "64B-2t2c-ethip4-ip4base-l3fwd-ndrdisc" \
     --xpath '//*[@framesize="64B" and contains(@tags,"BASE") and contains(@tags,"NDRDISC") and contains(@tags,"2T2C") and contains(@tags,"IP4FWD")]' --latency lat_50
 
+# HTML BUILDER
+sphinx-build -v -c . -a -b html -E \
+    -D release=$1 -D version="$1 report - $DATE" \
+    ${DIR[RST]} ${DIR[BUILD]}/
+
+# Patch the CSS for tables layout
+cat - > ${DIR[CSS_PATCH_FILE]} <<"_EOF"
+/* override table width restrictions */
+@media screen and (min-width: 767px) {
+    .wy-table-responsive table td, .wy-table-responsive table th {
+        white-space: normal !important;
+    }
+
+    .wy-table-responsive {
+        font-size: small;
+        margin-bottom: 24px;
+        max-width: 100%;
+        overflow: visible !important;
+    }
+}
+_EOF
+
+# LATEX BUILDER
+if [[ ${LATEX} -eq 1 ]] ;
+then
+    # Convert PyPLOT graphs in HTML format to PDF.
+    for f in ${DIR[STATIC,VPP]}/*; do
+        wkhtmltopdf ${f} ${f%.html}.pdf
+    done
+    for f in ${DIR[STATIC,DPDK]}/*; do
+        wkhtmltopdf ${f} ${f%.html}.pdf
+    done
+    rsvg-convert -z 10 -f pdf -o fdio.pdf fdio.svg
+
+    # Generate the LaTeX documentation
+    sphinx-build -v -c . -a -b latex -E \
+        -D release=$1 -D version="$1 report - $DATE" \
+        ${DIR[RST]} ${DIR[BUILD,LATEX]}
+    cd ${DIR[BUILD,LATEX]}
+    pdflatex -shell-escape -interaction nonstopmode csit.tex || true
+    pdflatex -interaction nonstopmode csit.tex || true
+    cp csit.pdf ../${DIR[STATIC,ARCH]}/csit_$1.pdf
+    cd ${SCRIPT_DIR}
+    rm -f fdio.pdf
+fi
+
 # Create archive
 echo Creating csit.report.tar.gz ...
 tar -czvf ./csit.report.tar.gz ${DIR[BUILD]}