JJB: Include perf analysis per job

+ Useful for performance analysis before run.
  This should help to analyze machine load before starting
  payload by comparing conditions from running jobs.
- Requirements: sysstat (apt install sysstat)
- Remove extraneous python import from
  backup_upload_archives.sh
- Fix sar issue on centos-7
- Fix debian9 lftools install / upgrade issue

Signed-off-by: pmikus <pmikus@cisco.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
Change-Id: I6b61f99b9b2cde2bac3dacdefaa926e9afbb157e

3 files changed, 41 insertions, 1 deletions

diff --git a/jjb/scripts/backup_upload_archives.sh b/jjb/scripts/backup_upload_archives.sh
index b49674914..b660f8687 100755
--- a/jjb/scripts/backup_upload_archives.sh
+++ b/jjb/scripts/backup_upload_archives.sh
@@ -35,7 +35,6 @@ cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
 import argparse
 import gzip
 import os
-import requests
 from mimetypes import MimeTypes
 
 from boto3 import resource
diff --git a/jjb/scripts/post_build_executor_info.sh b/jjb/scripts/post_build_executor_info.sh
index f98666676..fef79814e 100755
--- a/jjb/scripts/post_build_executor_info.sh
+++ b/jjb/scripts/post_build_executor_info.sh
@@ -45,3 +45,14 @@ if [ -n "$(which ccache)" ] ; then
 fi
 
 echo "$long_line"
+
+# Avoid sar anomaly on centos-7 in global-jjb/shell/logs-deploy.sh
+#
+# Note: VPP 20.09 will be removed in the next release cycle (21.10),
+#       therefore this hack is better than polluting the docker image
+#       build scripts with code to avoid installing sysstat on centos-7.
+#
+# TODO: Remove when vpp-*-2009-centos7-x86_64 jobs are removed
+if [ "$OS_ID" = "centos" ] && [ "$OS_VERSION_ID" = "7"  ] ; then
+    sudo yum remove -y sysstat >& /dev/null || true
+fi
diff --git a/jjb/scripts/setup_executor_env.sh b/jjb/scripts/setup_executor_env.sh
index a6b8216d6..4d3cb6ecf 100755
--- a/jjb/scripts/setup_executor_env.sh
+++ b/jjb/scripts/setup_executor_env.sh
@@ -25,6 +25,7 @@ file_delimiter="----- %< -----"
 long_line="************************************************************************"
 downloads_cache="/root/Downloads"
 
+# Node info
 echo "$long_line"
 echo "Executor Runtime Attributes:"
 echo "OS: $OS_ID-$OS_VERSION_ID"
@@ -41,6 +42,35 @@ else
     echo "Unknown Executor: '$dockerfile' not found!"
 fi
 
+# Performance analysis
+perf_trials=2
+perf_interval=1
+if [ "$OS_ID" == "ubuntu" ] || [ "$OS_ID" = "debian" ] ; then
+    SYSSTAT_PATH="/var/log/sysstat"
+elif [ "$OS_ID" == "centos" ] ; then
+    if [ "$OS_VERSION_ID" = "7" ] ; then
+        SYSSTAT_PATH="/var/log/sa/sa02"
+    else
+        SYSSTAT_PATH="/var/log/sa"
+    fi
+fi
+echo "$long_line"
+echo "Virtual memory stat"
+vmstat ${perf_interval} ${perf_trials}
+echo "CPU time breakdowns per CPU"
+mpstat -P ALL ${perf_interval}  ${perf_trials}
+echo "Per-process summary"
+pidstat ${perf_interval} ${perf_trials}
+echo "Block device stats"
+iostat -xz ${perf_interval} ${perf_trials}
+echo "Memory utilization"
+free -m
+echo "Network interface throughput"
+sar -n DEV -o ${SYSSTAT_PATH} ${perf_interval} ${perf_trials}
+echo "TCP metrics"
+sar -n TCP,ETCP -o ${SYSSTAT_PATH} ${perf_interval} ${perf_trials}
+
+# SW stack
 echo "$long_line"
 echo "Executor package list:"
 if [ "$OS_ID" == "ubuntu" ] || [ "$OS_ID" = "debian" ] ; then