Global: Docs generation in s3

+ Rework report/docs/trending jobs in CSIT
  to utilize AWS S3 cloudfront

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Ica2757ba8dc31792631c8d6b53b332ebf688b15e

6 files changed, 142 insertions, 156 deletions

diff --git a/jjb/scripts/csit/docs.sh b/jjb/scripts/csit/docs.sh
index 55048f2fd..9ecc78cd1 100644
--- a/jjb/scripts/csit/docs.sh
+++ b/jjb/scripts/csit/docs.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,64 +15,7 @@
 
 echo "---> jjb/scripts/csit/docs.sh"
 
-set -xe -o pipefail
+set -exuo pipefail
 
-[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site"
-[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit
-[ "$DOC_DIR" ] || DOC_DIR=resources/tools/doc_gen
-[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build
-[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site
-[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/doc
-[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn"
-
-cd ${DOC_DIR}
-chmod +x ./run_doc.sh
-./run_doc.sh ${GERRIT_BRANCH}
-
-retval=$?
-if [ ${retval} -ne "0" ]; then
-  echo "Documentation generation failed!"
-exit ${retval}
-fi
-
-if [[ ${JOB_NAME} == *merge* ]]; then
-
-  cd ${WORKSPACE}
-
-  mkdir -p ${RESOURCES_DIR}
-  mv -f ${BUILD_DIR}/* ${RESOURCES_DIR}
-  cd ${SITE_DIR}
-
-  cat > pom.xml << EOF
-  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>io.fd.csit</groupId>
-    <artifactId>docs</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-    <properties>
-      <generateReports>false</generateReports>
-    </properties>
-    <build>
-      <extensions>
-        <extension>
-          <groupId>org.apache.maven.wagon</groupId>
-          <artifactId>wagon-webdav-jackrabbit</artifactId>
-          <version>2.10</version>
-        </extension>
-      </extensions>
-    </build>
-    <distributionManagement>
-      <site>
-        <id>fdio-site</id>
-        <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url>
-      </site>
-    </distributionManagement>
-  </project>
-EOF
-
-  ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C
-
-  cd -
-
-fi
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_docs.sh"
diff --git a/jjb/scripts/csit/report.sh b/jjb/scripts/csit/report.sh
index 16018fdc9..0bd2e81bd 100644
--- a/jjb/scripts/csit/report.sh
+++ b/jjb/scripts/csit/report.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -15,64 +15,7 @@
 
 echo "---> jjb/scripts/csit/report.sh"
 
-set -xe -o pipefail
+set -exuo pipefail
 
-[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site"
-[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit
-[ "$DOC_DIR" ] || DOC_DIR=resources/tools/presentation
-[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build
-[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site
-[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/report
-[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn"
-
-cd ${DOC_DIR}
-chmod +x ./run_report.sh
-./run_report.sh ${GERRIT_BRANCH}
-
-retval=$?
-if [ ${retval} -ne "0" ]; then
-  echo "Report generation failed!"
-exit ${retval}
-fi
-
-if [[ ${JOB_NAME} == *merge* ]]; then
-
-  cd ${WORKSPACE}
-
-  mkdir -p ${RESOURCES_DIR}
-  mv -f ${BUILD_DIR}/* ${RESOURCES_DIR}
-  cd ${SITE_DIR}
-
-  cat > pom.xml << EOF
-  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>io.fd.csit</groupId>
-    <artifactId>docs</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-    <properties>
-      <generateReports>false</generateReports>
-    </properties>
-    <build>
-      <extensions>
-        <extension>
-          <groupId>org.apache.maven.wagon</groupId>
-          <artifactId>wagon-webdav-jackrabbit</artifactId>
-          <version>2.10</version>
-        </extension>
-      </extensions>
-    </build>
-    <distributionManagement>
-      <site>
-        <id>fdio-site</id>
-        <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url>
-      </site>
-    </distributionManagement>
-  </project>
-EOF
-
-  ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C
-
-  cd -
-
-fi
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_report.sh"
diff --git a/jjb/scripts/csit/trending.sh b/jjb/scripts/csit/trending.sh
new file mode 100644
index 000000000..715d15bf1
--- /dev/null
+++ b/jjb/scripts/csit/trending.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/csit/trending.sh"
+
+set -exuo pipefail
+
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_trending.sh"
diff --git a/jjb/scripts/publish_docs.sh b/jjb/scripts/publish_docs.sh
new file mode 100644
index 000000000..d6b843cbe
--- /dev/null
+++ b/jjb/scripts/publish_docs.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> publish_docs.sh"
+
+set -exuo pipefail
+
+S3_BUCKET="fdio-docs-s3-cloudfront-index"
+CDN_URL="s3-docs.fd.io"
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
+
+if [[ ${JOB_NAME} == *merge* ]]; then
+    case "${JOB_NAME}" in
+        *"csit-trending"*)
+            SITE_DIR="${WORKSPACE}/resources/tools/trending/_build"
+            s3_path="csit/${GERRIT_BRANCH}/trending"
+            ;;
+        *"csit-report"*)
+            SITE_DIR="${WORKSPACE}/resources/tools/presentation/_build"
+            s3_path="csit/${GERRIT_BRANCH}/report"
+            ;;
+        *"csit-docs"*)
+            SITE_DIR="${WORKSPACE}/resources/tools/doc_gen/_build"
+            s3_path="csit/${GERRIT_BRANCH}/docs"
+            ;;
+        *)
+            die "Unknown job: ${JOB_NAME}"
+    esac
+
+    echo "INFO: S3 path $s3_path"
+
+    echo "INFO: archiving docs to S3"
+    python3 $PYTHON_SCRIPT deploy_docs "$S3_BUCKET" "$s3_path" "$SITE_DIR"
+
+    echo "S3 docs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"
+fi
diff --git a/jjb/scripts/logs_publish.sh b/jjb/scripts/publish_library_py.sh
index da3593c66..f8430ff0b 100644
--- a/jjb/scripts/logs_publish.sh
+++ b/jjb/scripts/publish_library_py.sh
@@ -13,24 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-echo "---> logs_publish.sh"
+echo "---> publish_library_py.sh"
 
-CDN_URL="logs.nginx.service.consul"
-export AWS_ENDPOINT_URL="http://storage.service.consul:9000"
+set -exuo pipefail
 
-# FIXME: s3 config (until migrated to config provider, then pwd will be reset)
-mkdir -p ${HOME}/.aws
-echo "[default]
-aws_access_key_id = storage
-aws_secret_access_key = Storage1234" >> "$HOME/.aws/credentials"
-
-PYTHON_SCRIPT="/w/workspace/test-logs/logs_publish.py"
-
-# This script uploads the artifacts to a backup upload location
-if [ -f "$PYTHON_SCRIPT" ]; then
-    echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done"
-    exit 0
-fi
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
 
 pip3 install boto3
 mkdir -p $(dirname "$PYTHON_SCRIPT")
@@ -38,7 +25,7 @@ mkdir -p $(dirname "$PYTHON_SCRIPT")
 cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
 #!/usr/bin/python3
 
-"""Storage utilities library."""
+"""S3 publish library."""
 
 import gzip
 import logging
@@ -130,7 +117,7 @@ def upload(s3_resource, s3_bucket, src_fpath, s3_path):
     if not mime:
         mime = u"application/octet-stream"
 
-    if s3_bucket not in u"docs.fd.io":
+    if u"logs" in s3_bucket:
         if mime in COMPRESS_MIME and encoding != u"gzip":
             compress(src_fpath)
             src_fpath = src_fpath + u".gz"
@@ -178,6 +165,27 @@ def upload_recursive(s3_resource, s3_bucket, src_fpath, s3_path):
             )
 
 
+def deploy_docs(s3_bucket, s3_path, docs_dir):
+    """Ship docs dir content to S3 bucket. Requires the s3 bucket to exist.
+
+    :param s3_bucket: Name of S3 bucket. Eg: lf-project-date
+    :param s3_path: Path on S3 bucket to place the docs. Eg:
+        csit/${GERRIT_BRANCH}/report
+    :param docs_dir: Directory in which to recursively upload content.
+    :type s3_bucket: Object
+    :type s3_path: str
+    :type docs_dir: str
+    """
+    s3_resource = boto3.resource(u"s3")
+
+    upload_recursive(
+        s3_resource=s3_resource,
+        s3_bucket=s3_bucket,
+        src_fpath=docs_dir,
+        s3_path=s3_path
+    )
+
+
 def deploy_s3(s3_bucket, s3_path, build_url, workspace):
     """Add logs and archives to temp directory to be shipped to S3 bucket.
     Fetches logs and system information and pushes them and archives to S3
@@ -273,19 +281,3 @@ if __name__ == u"__main__":
     globals()[sys.argv[1]](*sys.argv[2:])
 
 END_OF_PYTHON_SCRIPT
-
-# The 'deploy_s3' command below expects the archives
-# directory to exist.  Normally lf-infra-sysstat or similar would
-# create it and add content, but to make sure this script is
-# self-contained, we ensure it exists here.
-mkdir -p "$WORKSPACE/archives"
-
-s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
-echo "INFO: S3 path $s3_path"
-
-echo "INFO: archiving backup logs to S3"
-# shellcheck disable=SC2086
-python3 $PYTHON_SCRIPT deploy_s3 "logs.fd.io" "$s3_path" \
-    "$BUILD_URL" "$WORKSPACE"
-
-echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"
diff --git a/jjb/scripts/publish_logs.sh b/jjb/scripts/publish_logs.sh
new file mode 100644
index 000000000..bc1e24c15
--- /dev/null
+++ b/jjb/scripts/publish_logs.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> publish_logs.sh"
+
+S3_BUCKET="logs.fd.io"
+CDN_URL="logs.nginx.service.consul"
+export AWS_ENDPOINT_URL="http://storage.service.consul:9000"
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
+
+# FIXME: s3 config (until migrated to config provider, then pwd will be reset)
+mkdir -p ${HOME}/.aws
+echo "[default]
+aws_access_key_id = storage
+aws_secret_access_key = Storage1234" > "$HOME/.aws/credentials"
+
+mkdir -p "$WORKSPACE/archives"
+
+s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
+
+echo "INFO: S3 path $s3_path"
+
+echo "INFO: archiving backup logs to S3"
+python3 $PYTHON_SCRIPT deploy_s3 "$S3_BUCKET" "$s3_path" \
+    "$BUILD_URL" "$WORKSPACE"
+
+echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"