Migrate report/trending/docs to bash framework

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Idf8462f15a2dd76b42dcaea9df6a0fc15a45503f

7 files changed, 349 insertions, 204 deletions

diff --git a/resources/libraries/bash/entry/bootstrap_docs.sh b/resources/libraries/bash/entry/bootstrap_docs.sh
new file mode 100755
index 0000000000..9d2519ebf3
--- /dev/null
+++ b/resources/libraries/bash/entry/bootstrap_docs.sh
@@ -0,0 +1,32 @@
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed."
+common_dirs || die
+activate_virtualenv || die
+generate_docs || die
+die_on_docs_error || die
diff --git a/resources/libraries/bash/entry/bootstrap_report.sh b/resources/libraries/bash/entry/bootstrap_report.sh
new file mode 100755
index 0000000000..191f910171
--- /dev/null
+++ b/resources/libraries/bash/entry/bootstrap_report.sh
@@ -0,0 +1,32 @@
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed."
+common_dirs || die
+activate_virtualenv || die
+generate_report || die
+die_on_docs_error || die
diff --git a/resources/libraries/bash/entry/bootstrap_trending.sh b/resources/libraries/bash/entry/bootstrap_trending.sh
new file mode 100755
index 0000000000..b172a81be5
--- /dev/null
+++ b/resources/libraries/bash/entry/bootstrap_trending.sh
@@ -0,0 +1,32 @@
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+# Assumptions:
+# + There is a directory holding CSIT code to use (this script is there).
+# Consequences (and specific assumptions) are multiple,
+# examine tree of functions for current description.
+
+# "set -eu" handles failures from the following two lines.
+BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+    echo "Source failed." >&2
+    exit 1
+}
+source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed."
+common_dirs || die
+activate_virtualenv || die
+generate_trending || die
+die_on_docs_error || die
diff --git a/resources/libraries/bash/function/docs.sh b/resources/libraries/bash/function/docs.sh
new file mode 100644
index 0000000000..ec5cbefdd0
--- /dev/null
+++ b/resources/libraries/bash/function/docs.sh
@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+
+function die_on_docs_error () {
+
+    # Source this fragment if you want to abort on any failure.
+    #
+    # Variables read:
+    # - DOCS_EXIT_STATUS - Set by a generation function.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    if [[ "${DOCS_EXIT_STATUS}" != "0" ]]; then
+        die "Failed to generate docs!" "${DOCS_EXIT_STATUS}"
+    fi
+}
+
+function generate_docs () {
+
+    # Generate docs content.
+    #
+    # Variable read:
+    # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools".
+    # Variables set:
+    # - DOCS_EXIT_STATUS - Exit status of docs generation.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${TOOLS_DIR}"/doc_gen || die "Pushd failed!"
+
+    WORKING_DIR="tmp"
+    BUILD_DIR="_build"
+
+    # Create working directories
+    mkdir "${BUILD_DIR}"
+    mkdir --parents "${WORKING_DIR}"/resources/libraries/python/
+    mkdir --parents "${WORKING_DIR}"/resources/libraries/robot/
+    mkdir --parents "${WORKING_DIR}"/tests/
+
+    # Copy the Sphinx source files:
+    cp -r src/* ${WORKING_DIR}/
+
+    # Copy the source files to be processed:
+    from_dir="../../../resources/libraries/python/"
+    to_dir="${WORKING_DIR}/resources/libraries/python/"
+    command="rsync -a --include '*/'"
+    ${command} --include '*.py' --exclude '*' "${from_dir}" "${to_dir}"
+    cp ../../../resources/__init__.py ${WORKING_DIR}/resources/
+    cp ../../../resources/libraries/__init__.py ${WORKING_DIR}/resources/libraries/
+    from_dir="../../../resources/libraries/robot/"
+    to_dir="${WORKING_DIR}/resources/libraries/robot/"
+    ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
+    from_dir="../../../tests/"
+    to_dir="${WORKING_DIR}/tests/"
+    ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}"
+
+    python3 gen_rst.py
+    # Remove all rst files from ./${WORKING_DIR}/env directory - we do not need
+    # them
+    find ./${WORKING_DIR}/env -type f -name '*.rst' | xargs rm -f
+
+    # Generate the documentation:
+    DATE=$(date -u '+%d-%b-%Y')
+
+    all_options=("-v")
+    all_options+=("-c" "${WORKING_DIR}")
+    all_options+=("-a")
+    all_options+=("-b" "html")
+    all_options+=("-E")
+    all_options+=("-D" "release=$1")
+    all_options+=("-D" "version='$1 documentation - $DATE'")
+    all_options+=("${WORKING_DIR}" "${BUILD_DIR}/")
+
+    set +e
+    sphinx-build "${all_options[@]}"
+    DOCS_EXIT_STATUS="$?"
+    set -e
+
+    find . -type d -name 'env' | xargs rm -rf
+
+}
+
+function generate_report () {
+
+    # Generate report content.
+    #
+    # Variable read:
+    # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools".
+    # - ${GERRIT_BRANCH} - Gerrit branch used for release tagging.
+    # Variables set:
+    # - DOCS_EXIT_STATUS - Exit status of report generation.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!"
+
+    # Set default values in config array.
+    typeset -A CFG
+    typeset -A DIR
+
+    DIR[WORKING]="_tmp"
+
+    # Create working directories.
+    mkdir "${DIR[WORKING]}" || die "Mkdir failed!"
+
+    export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!"
+
+    all_options=("pal.py")
+    all_options+=("--specification" "specifications/report")
+    all_options+=("--release" "${GERRIT_BRANCH:-master}")
+    all_options+=("--week" $(date "+%V"))
+    all_options+=("--logging" "INFO")
+    all_options+=("--force")
+
+    set +e
+    python "${all_options[@]}"
+    DOCS_EXIT_STATUS="$?"
+    set -e
+
+}
+
+function generate_report_local () {
+
+    # Generate report from local content.
+    #
+    # Variable read:
+    # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools".
+    # - ${CSIT_REPORT_FILENAME} - Source filename.
+    # - ${CSIT_REPORT_DIRECTORYNAME} - Source directory.
+    # - ${CSIT_REPORT_INSTALL_DEPENDENCIES} - Whether to install dependencies.
+    # - ${CSIT_REPORT_INSTALL_LATEX} - Whether to install latex.
+    # Variables set:
+    # - DOCS_EXIT_STATUS - Exit status of report generation.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!"
+
+    filename="${CSIT_REPORT_FILENAME-}"
+    directoryname="${CSIT_REPORT_DIRECTORYNAME-}"
+    install_dependencies="${CSIT_REPORT_INSTALL_DEPENDENCIES:-1}"
+    install_latex="${CSIT_REPORT_INSTALL_LATEX:-0}"
+
+    # Set default values in config array.
+    typeset -A CFG
+    typeset -A DIR
+
+    DIR[WORKING]="_tmp"
+
+    # Install system dependencies.
+    if [[ ${install_dependencies} -eq 1 ]] ;
+    then
+        sudo apt -y update || die "APT update failed!"
+        sudo apt -y install libxml2 libxml2-dev libxslt-dev \
+            build-essential zlib1g-dev unzip || die "APT install failed!"
+    fi
+
+    if [[ ${install_latex} -eq 1 ]] ;
+    then
+        sudo apt -y update || die "APT update failed!"
+        sudo apt -y install xvfb texlive-latex-recommended \
+            texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra \
+            latexmk wkhtmltopdf inkscape || die "APT install failed!"
+        target="/usr/share/texlive/texmf-dist/web2c/texmf.cnf"
+        sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' "${target}" || {
+            die "Patching latex failed!"
+        }
+    fi
+
+    # Create working directories.
+    mkdir "${DIR[WORKING]}" || die "Mkdir failed!"
+
+    export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!"
+
+    all_options=("pal.py")
+    all_options+=("--specification" "specifications/report_local")
+    all_options+=("--release" "${RELEASE:-master}")
+    all_options+=("--week" "${WEEK:-1}")
+    all_options+=("--logging" "INFO")
+    all_options+=("--force")
+    if [[ ${filename} != "" ]]; then
+        all_options+=("--input-file" "${filename}")
+    fi
+    if [[ ${directoryname} != "" ]]; then
+        all_options+=("--input-directory" "${directoryname}")
+    fi
+
+    set +e
+    python "${all_options[@]}"
+    DOCS_EXIT_STATUS="$?"
+    set -e
+
+}
+
+function generate_trending () {
+
+    # Generate trending content.
+    #
+    # Variable read:
+    # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools".
+    # Variables set:
+    # - DOCS_EXIT_STATUS - Exit status of trending generation.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!"
+
+    # Set default values in config array.
+    typeset -A DIR
+
+    DIR[WORKING]="_tmp"
+
+    # Create working directories.
+    mkdir "${DIR[WORKING]}" || die "Mkdir failed!"
+
+    export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!"
+
+    all_options=("pal.py")
+    all_options+=("--specification" "specifications/trending")
+    all_options+=("--logging" "INFO")
+    all_options+=("--force")
+
+    set +e
+    python "${all_options[@]}"
+    DOCS_EXIT_STATUS="$?"
+    set -e
+
+}
diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh
deleted file mode 100755
index 842339f7f5..0000000000
--- a/resources/tools/presentation/run_cpta.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# set default values in config array
-typeset -A DIR
-
-DIR[WORKING]=_tmp
-
-# Create working directories
-mkdir ${DIR[WORKING]}
-
-# Create virtual environment
-virtualenv -p $(which python3) ${DIR[WORKING]}/env
-source ${DIR[WORKING]}/env/bin/activate
-
-# FIXME: s3 config (until migrated to vault, then account will be reset)
-mkdir -p ${HOME}/.aws
-echo "[nomad-s3]" >> ${HOME}/.aws/config
-echo "[nomad-s3]
-aws_access_key_id = csit
-aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials
-
-# Install python dependencies:
-pip3 install -r requirements.txt
-
-export PYTHONPATH=`pwd`:`pwd`/../../../
-
-STATUS=$(python pal.py \
-    --specification specifications/trending \
-    --logging INFO \
-    --force)
-RETURN_STATUS=$?
-
-echo ${STATUS}
-exit ${RETURN_STATUS}
diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh
deleted file mode 100755
index d1994a1a27..0000000000
--- a/resources/tools/presentation/run_report.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -x
-
-RELEASE=$1
-
-# set default values in config array
-typeset -A CFG
-typeset -A DIR
-
-DIR[WORKING]=_tmp
-
-# Create working directories
-mkdir ${DIR[WORKING]}
-
-# Create virtual environment
-virtualenv -p $(which python3) ${DIR[WORKING]}/env
-source ${DIR[WORKING]}/env/bin/activate
-
-# FIXME: s3 config (until migrated to vault, then account will be reset)
-mkdir -p ${HOME}/.aws
-echo "[nomad-s3]" >> ${HOME}/.aws/config
-echo "[nomad-s3]
-aws_access_key_id = csit
-aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials
-
-# Install python dependencies:
-pip3 install -r requirements.txt
-
-export PYTHONPATH=`pwd`:`pwd`/../../../
-
-python pal.py \
-    --specification specifications/report \
-    --release ${RELEASE} \
-    --week "33" \
-    --logging INFO \
-    --force
-
-RETURN_STATUS=$(echo $?)
-exit ${RETURN_STATUS}
diff --git a/resources/tools/presentation/run_report_local.sh b/resources/tools/presentation/run_report_local.sh
deleted file mode 100755
index df0e89e5e5..0000000000
--- a/resources/tools/presentation/run_report_local.sh
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/bin/bash
-
-# set -x
-
-usage()
-{
-  cat <<EOF
-usage: run_report_local [OPTION]
-
-Options:
-  -h; --help             Display this help and exit.
-  -f; --file             Input XML file to be processed.
-  -d; --directory        Directory with input XML files to be processed.
-  -r; --release          Release string (optional).
-  -w; --week             Release week (optional).
-  -i; --no-dependencies  Do not install dependencies.
-  -l; --install-latex    Instal Latex.
-EOF
-}
-
-filename=""
-directoryname=""
-release="master"
-week="1"
-cfg_install_dependencies=1
-cfg_install_latex=0
-
-while [ "$1" != "" ]; do
-    case $1 in
-        -f | --file )               shift
-                                    filename=$1
-                                    ;;
-        -d | --directory )          shift
-                                    directoryname=$1
-                                    ;;
-        -r | --release )            shift
-                                    release=$1
-                                    ;;
-        -w | --week )               shift
-                                    week=$1
-                                    ;;
-        -i | --no-dependencies )    cfg_install_dependencies=0
-                                    ;;
-        -l | --install-latex )      cfg_install_latex=1
-                                    ;;
-        -h | --help )               usage
-                                    exit 1
-                                    ;;
-        * )                         usage
-                                    exit 1
-    esac
-    shift
-done
-
-echo "Parameters:"
-echo "  Input file:           " ${filename}
-echo "  Input directory:      " ${directoryname}
-echo "  Report release:       " ${release}
-echo "  Report week:          " ${week}
-echo "  Install dependencies: " ${cfg_install_dependencies}
-echo "  Install Latex:        " ${cfg_install_latex}
-
-if [[ $filename == "" && $directoryname == "" ]]; then
-    echo "ERROR: The input directory or file is required."
-    usage
-    exit 1
-fi
-
-# set default values in config array
-typeset -A CFG
-typeset -A DIR
-
-DIR[WORKING]=_tmp
-
-# Install system dependencies
-if [[ ${cfg_install_dependencies} -eq 1 ]] ;
-then
-sudo apt-get -y update
-sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \
-    zlib1g-dev unzip
-fi
-
-if [[ ${cfg_install_latex} -eq 1 ]] ;
-then
-    sudo apt-get -y install xvfb texlive-latex-recommended \
-        texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra \
-        latexmk wkhtmltopdf inkscape
-    target="/usr/share/texlive/texmf-dist/web2c/texmf.cnf"
-    sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' "${target}"
-fi
-
-# Create working directories
-mkdir ${DIR[WORKING]}
-
-# Create virtual environment
-virtualenv -p $(which python3) ${DIR[WORKING]}/env
-source ${DIR[WORKING]}/env/bin/activate
-
-# Install python dependencies:
-pip3 install -r requirements.txt
-
-export PYTHONPATH=`pwd`:`pwd`/../../../:`pwd`/../../libraries/python
-
-# Show help so you know the meaning of all parameters
-python pal.py --help
-
-if [[ ${filename} != "" ]]; then
-    python pal.py \
-        --specification specifications/report_local \
-        --release ${release} \
-        --week ${week} \
-        --logging INFO \
-        --force \
-        --input-file ${filename}
-fi
-
-if [[ ${directoryname} != "" ]]; then
-    python pal.py \
-        --specification specifications/report_local \
-        --release ${release} \
-        --week ${week} \
-        --logging INFO \
-        --force \
-        --input-directory ${directoryname}
-fi
-
-RETURN_STATUS=$(echo $?)
-exit ${RETURN_STATUS}