diff options
| -rwxr-xr-x | resources/libraries/bash/entry/bootstrap_docs.sh | 32 | ||||
| -rwxr-xr-x | resources/libraries/bash/entry/bootstrap_report.sh | 32 | ||||
| -rwxr-xr-x | resources/libraries/bash/entry/bootstrap_trending.sh | 32 | ||||
| -rw-r--r-- | resources/libraries/bash/function/docs.sh | 253 | ||||
| -rwxr-xr-x | resources/tools/presentation/run_cpta.sh | 36 | ||||
| -rwxr-xr-x | resources/tools/presentation/run_report.sh | 40 | ||||
| -rwxr-xr-x | resources/tools/presentation/run_report_local.sh | 126 |
7 files changed, 349 insertions, 202 deletions
diff --git a/resources/libraries/bash/entry/bootstrap_docs.sh b/resources/libraries/bash/entry/bootstrap_docs.sh new file mode 100755 index 0000000000..9d2519ebf3 --- /dev/null +++ b/resources/libraries/bash/entry/bootstrap_docs.sh @@ -0,0 +1,32 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed." +common_dirs || die +activate_virtualenv || die +generate_docs || die +die_on_docs_error || die diff --git a/resources/libraries/bash/entry/bootstrap_report.sh b/resources/libraries/bash/entry/bootstrap_report.sh new file mode 100755 index 0000000000..191f910171 --- /dev/null +++ b/resources/libraries/bash/entry/bootstrap_report.sh @@ -0,0 +1,32 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed." +common_dirs || die +activate_virtualenv || die +generate_report || die +die_on_docs_error || die diff --git a/resources/libraries/bash/entry/bootstrap_trending.sh b/resources/libraries/bash/entry/bootstrap_trending.sh new file mode 100755 index 0000000000..b172a81be5 --- /dev/null +++ b/resources/libraries/bash/entry/bootstrap_trending.sh @@ -0,0 +1,32 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +# Assumptions: +# + There is a directory holding CSIT code to use (this script is there). +# Consequences (and specific assumptions) are multiple, +# examine tree of functions for current description. + +# "set -eu" handles failures from the following two lines. +BASH_ENTRY_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_ENTRY_DIR}/../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} +source "${BASH_FUNCTION_DIR}/docs.sh" || die "Source failed." +common_dirs || die +activate_virtualenv || die +generate_trending || die +die_on_docs_error || die diff --git a/resources/libraries/bash/function/docs.sh b/resources/libraries/bash/function/docs.sh new file mode 100644 index 0000000000..ec5cbefdd0 --- /dev/null +++ b/resources/libraries/bash/function/docs.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + + +function die_on_docs_error () { + + # Source this fragment if you want to abort on any failure. + # + # Variables read: + # - DOCS_EXIT_STATUS - Set by a generation function. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + if [[ "${DOCS_EXIT_STATUS}" != "0" ]]; then + die "Failed to generate docs!" "${DOCS_EXIT_STATUS}" + fi +} + +function generate_docs () { + + # Generate docs content. + # + # Variable read: + # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools". + # Variables set: + # - DOCS_EXIT_STATUS - Exit status of docs generation. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/doc_gen || die "Pushd failed!" + + WORKING_DIR="tmp" + BUILD_DIR="_build" + + # Create working directories + mkdir "${BUILD_DIR}" + mkdir --parents "${WORKING_DIR}"/resources/libraries/python/ + mkdir --parents "${WORKING_DIR}"/resources/libraries/robot/ + mkdir --parents "${WORKING_DIR}"/tests/ + + # Copy the Sphinx source files: + cp -r src/* ${WORKING_DIR}/ + + # Copy the source files to be processed: + from_dir="../../../resources/libraries/python/" + to_dir="${WORKING_DIR}/resources/libraries/python/" + command="rsync -a --include '*/'" + ${command} --include '*.py' --exclude '*' "${from_dir}" "${to_dir}" + cp ../../../resources/__init__.py ${WORKING_DIR}/resources/ + cp ../../../resources/libraries/__init__.py ${WORKING_DIR}/resources/libraries/ + from_dir="../../../resources/libraries/robot/" + to_dir="${WORKING_DIR}/resources/libraries/robot/" + ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}" + from_dir="../../../tests/" + to_dir="${WORKING_DIR}/tests/" + ${command} --include '*.robot' --exclude '*' "${from_dir}" "${to_dir}" + + python3 gen_rst.py + # Remove all rst files from ./${WORKING_DIR}/env directory - we do not need + # them + find ./${WORKING_DIR}/env -type f -name '*.rst' | xargs rm -f + + # Generate the documentation: + DATE=$(date -u '+%d-%b-%Y') + + all_options=("-v") + all_options+=("-c" "${WORKING_DIR}") + all_options+=("-a") + all_options+=("-b" "html") + all_options+=("-E") + all_options+=("-D" "release=$1") + all_options+=("-D" "version='$1 documentation - $DATE'") + all_options+=("${WORKING_DIR}" "${BUILD_DIR}/") + + set +e + sphinx-build "${all_options[@]}" + DOCS_EXIT_STATUS="$?" + set -e + + find . -type d -name 'env' | xargs rm -rf + +} + +function generate_report () { + + # Generate report content. + # + # Variable read: + # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools". + # - ${GERRIT_BRANCH} - Gerrit branch used for release tagging. + # Variables set: + # - DOCS_EXIT_STATUS - Exit status of report generation. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!" + + # Set default values in config array. + typeset -A CFG + typeset -A DIR + + DIR[WORKING]="_tmp" + + # Create working directories. + mkdir "${DIR[WORKING]}" || die "Mkdir failed!" + + export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!" + + all_options=("pal.py") + all_options+=("--specification" "specifications/report") + all_options+=("--release" "${GERRIT_BRANCH:-master}") + all_options+=("--week" $(date "+%V")) + all_options+=("--logging" "INFO") + all_options+=("--force") + + set +e + python "${all_options[@]}" + DOCS_EXIT_STATUS="$?" + set -e + +} + +function generate_report_local () { + + # Generate report from local content. + # + # Variable read: + # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools". + # - ${CSIT_REPORT_FILENAME} - Source filename. + # - ${CSIT_REPORT_DIRECTORYNAME} - Source directory. + # - ${CSIT_REPORT_INSTALL_DEPENDENCIES} - Whether to install dependencies. + # - ${CSIT_REPORT_INSTALL_LATEX} - Whether to install latex. + # Variables set: + # - DOCS_EXIT_STATUS - Exit status of report generation. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!" + + filename="${CSIT_REPORT_FILENAME-}" + directoryname="${CSIT_REPORT_DIRECTORYNAME-}" + install_dependencies="${CSIT_REPORT_INSTALL_DEPENDENCIES:-1}" + install_latex="${CSIT_REPORT_INSTALL_LATEX:-0}" + + # Set default values in config array. + typeset -A CFG + typeset -A DIR + + DIR[WORKING]="_tmp" + + # Install system dependencies. + if [[ ${install_dependencies} -eq 1 ]] ; + then + sudo apt -y update || die "APT update failed!" + sudo apt -y install libxml2 libxml2-dev libxslt-dev \ + build-essential zlib1g-dev unzip || die "APT install failed!" + fi + + if [[ ${install_latex} -eq 1 ]] ; + then + sudo apt -y update || die "APT update failed!" + sudo apt -y install xvfb texlive-latex-recommended \ + texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra \ + latexmk wkhtmltopdf inkscape || die "APT install failed!" + target="/usr/share/texlive/texmf-dist/web2c/texmf.cnf" + sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' "${target}" || { + die "Patching latex failed!" + } + fi + + # Create working directories. + mkdir "${DIR[WORKING]}" || die "Mkdir failed!" + + export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!" + + all_options=("pal.py") + all_options+=("--specification" "specifications/report_local") + all_options+=("--release" "${RELEASE:-master}") + all_options+=("--week" "${WEEK:-1}") + all_options+=("--logging" "INFO") + all_options+=("--force") + if [[ ${filename} != "" ]]; then + all_options+=("--input-file" "${filename}") + fi + if [[ ${directoryname} != "" ]]; then + all_options+=("--input-directory" "${directoryname}") + fi + + set +e + python "${all_options[@]}" + DOCS_EXIT_STATUS="$?" + set -e + +} + +function generate_trending () { + + # Generate trending content. + # + # Variable read: + # - ${TOOLS_DIR} - Path to existing resources subdirectory "tools". + # Variables set: + # - DOCS_EXIT_STATUS - Exit status of trending generation. + # Functions called: + # - die - Print to stderr and exit. + + set -exuo pipefail + + pushd "${TOOLS_DIR}"/presentation || die "Pushd failed!" + + # Set default values in config array. + typeset -A DIR + + DIR[WORKING]="_tmp" + + # Create working directories. + mkdir "${DIR[WORKING]}" || die "Mkdir failed!" + + export PYTHONPATH=`pwd`:`pwd`/../../../ || die "Export failed!" + + all_options=("pal.py") + all_options+=("--specification" "specifications/trending") + all_options+=("--logging" "INFO") + all_options+=("--force") + + set +e + python "${all_options[@]}" + DOCS_EXIT_STATUS="$?" + set -e + +} diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh deleted file mode 100755 index 842339f7f5..0000000000 --- a/resources/tools/presentation/run_cpta.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -x - -# set default values in config array -typeset -A DIR - -DIR[WORKING]=_tmp - -# Create working directories -mkdir ${DIR[WORKING]} - -# Create virtual environment -virtualenv -p $(which python3) ${DIR[WORKING]}/env -source ${DIR[WORKING]}/env/bin/activate - -# FIXME: s3 config (until migrated to vault, then account will be reset) -mkdir -p ${HOME}/.aws -echo "[nomad-s3]" >> ${HOME}/.aws/config -echo "[nomad-s3] -aws_access_key_id = csit -aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials - -# Install python dependencies: -pip3 install -r requirements.txt - -export PYTHONPATH=`pwd`:`pwd`/../../../ - -STATUS=$(python pal.py \ - --specification specifications/trending \ - --logging INFO \ - --force) -RETURN_STATUS=$? - -echo ${STATUS} -exit ${RETURN_STATUS} diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh deleted file mode 100755 index d1994a1a27..0000000000 --- a/resources/tools/presentation/run_report.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -x - -RELEASE=$1 - -# set default values in config array -typeset -A CFG -typeset -A DIR - -DIR[WORKING]=_tmp - -# Create working directories -mkdir ${DIR[WORKING]} - -# Create virtual environment -virtualenv -p $(which python3) ${DIR[WORKING]}/env -source ${DIR[WORKING]}/env/bin/activate - -# FIXME: s3 config (until migrated to vault, then account will be reset) -mkdir -p ${HOME}/.aws -echo "[nomad-s3]" >> ${HOME}/.aws/config -echo "[nomad-s3] -aws_access_key_id = csit -aws_secret_access_key = Csit1234" >> ${HOME}/.aws/credentials - -# Install python dependencies: -pip3 install -r requirements.txt - -export PYTHONPATH=`pwd`:`pwd`/../../../ - -python pal.py \ - --specification specifications/report \ - --release ${RELEASE} \ - --week "33" \ - --logging INFO \ - --force - -RETURN_STATUS=$(echo $?) -exit ${RETURN_STATUS} diff --git a/resources/tools/presentation/run_report_local.sh b/resources/tools/presentation/run_report_local.sh deleted file mode 100755 index b48d4d2db5..0000000000 --- a/resources/tools/presentation/run_report_local.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -# set -x - -usage() -{ - cat <<EOF -usage: run_report_local [OPTION] - -Options: - -h; --help Display this help and exit. - -f; --file Input XML file to be processed. - -d; --directory Directory with input XML files to be processed. - -r; --release Release string (optional). - -w; --week Release week (optional). - -i; --no-dependencies Do not install dependencies. - -l; --install-latex Instal Latex. -EOF -} - -filename="" -directoryname="" -release="master" -week="1" -cfg_install_dependencies=1 -cfg_install_latex=0 - -while [ "$1" != "" ]; do - case $1 in - -f | --file ) shift - filename=$1 - ;; - -d | --directory ) shift - directoryname=$1 - ;; - -r | --release ) shift - release=$1 - ;; - -w | --week ) shift - week=$1 - ;; - -i | --no-dependencies ) cfg_install_dependencies=0 - ;; - -l | --install-latex ) cfg_install_latex=1 - ;; - -h | --help ) usage - exit 1 - ;; - * ) usage - exit 1 - esac - shift -done - -echo "Parameters:" -echo " Input file: " ${filename} -echo " Input directory: " ${directoryname} -echo " Report release: " ${release} -echo " Report week: " ${week} -echo " Install dependencies: " ${cfg_install_dependencies} -echo " Install Latex: " ${cfg_install_latex} - -if [[ $filename == "" && $directoryname == "" ]]; then - echo "ERROR: The input directory or file is required." - usage - exit 1 -fi - -# set default values in config array -typeset -A CFG -typeset -A DIR - -DIR[WORKING]=_tmp - -# Install system dependencies -if [[ ${cfg_install_dependencies} -eq 1 ]] ; -then -sudo apt-get -y update -sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \ - zlib1g-dev unzip -fi - -if [[ ${cfg_install_latex} -eq 1 ]] ; -then - sudo apt-get -y install xvfb texlive-latex-recommended \ - texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra latexmk wkhtmltopdf inkscape - sudo sed -i.bak 's/^\(main_memory\s=\s\).*/\110000000/' /usr/share/texlive/texmf-dist/web2c/texmf.cnf -fi - -# Create working directories -mkdir ${DIR[WORKING]} - -# Create virtual environment -virtualenv -p $(which python3) ${DIR[WORKING]}/env -source ${DIR[WORKING]}/env/bin/activate - -# Install python dependencies: -pip3 install -r requirements.txt - -export PYTHONPATH=`pwd`:`pwd`/../../../:`pwd`/../../libraries/python - -# Show help so you know the meaning of all parameters -python pal.py --help - -if [[ ${filename} != "" ]]; then - python pal.py \ - --specification specifications/report_local \ - --release ${release} \ - --week ${week} \ - --logging INFO \ - --force \ - --input-file ${filename} -fi - -if [[ ${directoryname} != "" ]]; then - python pal.py \ - --specification specifications/report_local \ - --release ${release} \ - --week ${week} \ - --logging INFO \ - --force \ - --input-directory ${directoryname} -fi - -RETURN_STATUS=$(echo $?) -exit ${RETURN_STATUS} |