feat(scripts): Introduce console log analyzer

This semi-automatic script saves time for RCA after release.

Change-Id: Idda3de00635b3c16b71df024616f1e2e94d7c1d6
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

1 files changed, 60 insertions, 0 deletions

diff --git a/resources/tools/scripts/rca_console_logs.sh b/resources/tools/scripts/rca_console_logs.sh
new file mode 100644
index 0000000000..58d61a1818
--- /dev/null
+++ b/resources/tools/scripts/rca_console_logs.sh
@@ -0,0 +1,60 @@
+# Copyright (c) 2024 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+set +x
+
+# This tool saves time for RCA after release.
+
+# First argument: Jenkins job to analyze.
+# Second argument: Pattern to looks for (e.g. identifying release instead of RC2).
+
+# Example usage:
+# bash rca_console_logs.sh 'https://s3-logs.fd.io/vex-yul-rot-jenkins-1/csit-vpp-perf-report-iterative-2410-2n-spr' '24.10-release'
+
+# For each run, this script prints hints on whether skip or look deeper.
+# Also testbeds are printed, to see possible correlations with failures.
+
+jobname="${1}"
+build_pattern="${2}"
+skip_before="${3-1}"
+# TODO: Detect last run and go backward?
+for i in {1..999}; do
+    if (( ${i} < ${skip_before} )); then
+        # Silently skip.
+        continue
+    fi
+    if ! curl -sf "${jobname}/${i}/console.log.gz" | zcat > "console.log"; then
+        echo "${i}: failed to download. No more runs?"
+        exit 0
+    fi
+    if ! fgrep -q "${build_pattern}" "console.log"; then
+        echo "${i}: not matching the pattern, skip."
+        continue
+    fi
+    if ! grep '.* tests, .* passed, .* failed' "console.log" > "tests.txt"; then
+        echo "${i}: no tests run? suspicious."
+        continue
+    fi
+    final=$(tail -1 "tests.txt" | tee "final.txt")
+    if fgrep -q ', 0 failed' "final.txt"; then
+        echo -ne "${i}: skip ${final}\t\t"
+    else
+        echo -ne "${i}: investigate ${final}\t\t"
+    fi
+    # TODO: Simplify this topology detection.
+    line=$(grep 'TOPOLOGY_PATH:' "console.log")
+    topology_path=`echo "$line" | sed -n 's/.*TOPOLOGY_PATH:\([^ ]*\).*/\1/p'`
+    topology_name=$(basename "$topology_path" | sed 's/\.[^.]*$//')
+    echo "$topology_name"
+done