hs-test: generate core dump, fix docker logs in CI

Type: test

Change-Id: Ie1f66cdc061d3eccefc2ce58e977d88a33340038
Signed-off-by: Adrian Villin <avillin@cisco.com>

3 files changed, 109 insertions, 21 deletions

diff --git a/extras/hs-test/infra/container.go b/extras/hs-test/infra/container.go
index d8bddab211e..195f889f237 100644
--- a/extras/hs-test/infra/container.go
+++ b/extras/hs-test/infra/container.go
@@ -14,6 +14,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/cilium/cilium/pkg/sysctl"
 	containerTypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/image"
@@ -325,6 +326,15 @@ func (c *Container) getVolumesAsSlice() []string {
 		volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", *VppSourceFileDir, *VppSourceFileDir))
 	}
 
+	core_pattern, err := sysctl.Read("kernel.core_pattern")
+	if err == nil {
+		index := strings.LastIndex(core_pattern, "/")
+		core_pattern = core_pattern[:index]
+		volumeSlice = append(volumeSlice, c.Suite.getLogDirPath()+":"+core_pattern)
+	} else {
+		c.Suite.Log(err)
+	}
+
 	if len(c.Volumes) > 0 {
 		for _, volume := range c.Volumes {
 			volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", volume.HostDir, volume.ContainerDir))
@@ -435,21 +445,8 @@ func (c *Container) Exec(command string, arguments ...any) string {
 	return string(byteOutput)
 }
 
-func (c *Container) getLogDirPath() string {
-	testId := c.Suite.GetTestId()
-	testName := c.Suite.GetCurrentTestName()
-	logDirPath := logDir + testName + "/" + testId + "/"
-
-	cmd := exec.Command("mkdir", "-p", logDirPath)
-	if err := cmd.Run(); err != nil {
-		Fail("mkdir error: " + fmt.Sprint(err))
-	}
-
-	return logDirPath
-}
-
 func (c *Container) saveLogs() {
-	testLogFilePath := c.getLogDirPath() + "container-" + c.Name + ".log"
+	testLogFilePath := c.Suite.getLogDirPath() + "container-" + c.Name + ".log"
 
 	logs, err := c.log(0)
 	if err != nil {
diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go
index ff5b02eb95b..271ef99f249 100644
--- a/extras/hs-test/infra/hst_suite.go
+++ b/extras/hs-test/infra/hst_suite.go
@@ -38,6 +38,7 @@ var VppSourceFileDir = flag.String("vppsrc", "", "vpp source file directory")
 var IsDebugBuild = flag.Bool("debug_build", false, "some paths are different with debug build")
 var UseCpu0 = flag.Bool("cpu0", false, "use cpu0")
 var IsLeakCheck = flag.Bool("leak_check", false, "run leak-check tests")
+var ParallelTotal = flag.Lookup("ginkgo.parallel.total")
 var NumaAwareCpuAlloc bool
 var SuiteTimeout time.Duration
 
@@ -59,11 +60,39 @@ type HstSuite struct {
 	Docker            *client.Client
 }
 
+// used for colorful ReportEntry
+type StringerStruct struct {
+	Label string
+}
+
+// ColorableString for ReportEntry to use
+func (s StringerStruct) ColorableString() string {
+	return fmt.Sprintf("{{red}}%s{{/}}", s.Label)
+}
+
+// non-colorable String() is used by go's string formatting support but ignored by ReportEntry
+func (s StringerStruct) String() string {
+	return s.Label
+}
+
 func getTestFilename() string {
 	_, filename, _, _ := runtime.Caller(2)
 	return filepath.Base(filename)
 }
 
+func (s *HstSuite) getLogDirPath() string {
+	testId := s.GetTestId()
+	testName := s.GetCurrentTestName()
+	logDirPath := logDir + testName + "/" + testId + "/"
+
+	cmd := exec.Command("mkdir", "-p", logDirPath)
+	if err := cmd.Run(); err != nil {
+		Fail("mkdir error: " + fmt.Sprint(err))
+	}
+
+	return logDirPath
+}
+
 func (s *HstSuite) newDockerClient() {
 	var err error
 	s.Docker, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
@@ -117,6 +146,7 @@ func (s *HstSuite) TearDownTest() {
 	if *IsPersistent {
 		return
 	}
+	s.WaitForCoreDump()
 	s.ResetContainers()
 
 	if s.Ip4AddrAllocator != nil {
@@ -183,7 +213,7 @@ func (s *HstSuite) HstFail() {
 		out, err := container.log(20)
 		if err != nil {
 			s.Log("An error occured while obtaining '" + container.Name + "' container logs: " + fmt.Sprint(err))
-			s.Log("The container might not be running - check logs in " + container.getLogDirPath())
+			s.Log("The container might not be running - check logs in " + s.getLogDirPath())
 			continue
 		}
 		s.Log("\nvvvvvvvvvvvvvvv " +
@@ -297,6 +327,71 @@ func (s *HstSuite) SkipUnlessLeakCheck() {
 		s.Skip("leak-check tests excluded")
 	}
 }
+
+func (s *HstSuite) WaitForCoreDump() {
+	var filename string
+	var cmd *exec.Cmd
+	dir, err := os.Open(s.getLogDirPath())
+	if err != nil {
+		s.Log(err)
+		return
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdirnames(0)
+	if err != nil {
+		s.Log(err)
+		return
+	}
+	for _, file := range files {
+		if strings.Contains(file, "core") {
+			filename = file
+		}
+	}
+	timeout := 60
+	waitTime := 5
+
+	if filename != "" {
+		filename := s.getLogDirPath() + filename
+		s.Log(fmt.Sprintf("WAITING FOR CORE DUMP (%s)", filename))
+		for i := waitTime; i <= timeout; i += waitTime {
+			fileInfo, err := os.Stat(filename)
+			if err != nil {
+				s.Log("Error while reading file info: " + fmt.Sprint(err))
+				return
+			}
+			currSize := fileInfo.Size()
+			s.Log(fmt.Sprintf("Waiting %ds/%ds...", i, timeout))
+			time.Sleep(time.Duration(waitTime) * time.Second)
+			fileInfo, _ = os.Stat(filename)
+
+			if currSize == fileInfo.Size() {
+				if *IsDebugBuild {
+					cmd = exec.Command("sudo", "gdb", "../../build-root/build-vpp_debug-native/vpp/bin/vpp",
+						"-c", filename, "-ex", "bt", "full", "-ex", "quit")
+				} else {
+					cmd = exec.Command("sudo", "gdb", "../../build-root/build-vpp-native/vpp/bin/vpp",
+						"-c", filename, "-ex", "bt", "full", "-ex", "quit")
+				}
+
+				s.Log(cmd)
+				output, _ := cmd.Output()
+				AddReportEntry("VPP Backtrace", StringerStruct{Label: string(output)})
+				os.WriteFile(s.getLogDirPath()+"backtrace.log", output, os.FileMode(0644))
+				if s.CpuAllocator.runningInCi {
+					err = os.Remove(filename)
+					if err == nil {
+						s.Log("removed " + filename)
+					} else {
+						s.Log(err)
+					}
+				}
+				return
+			}
+		}
+	}
+}
+
 func (s *HstSuite) ResetContainers() {
 	for _, container := range s.StartedContainers {
 		container.stop()
diff --git a/extras/hs-test/infra/vppinstance.go b/extras/hs-test/infra/vppinstance.go
index dfb236b6725..a93921ed5be 100644
--- a/extras/hs-test/infra/vppinstance.go
+++ b/extras/hs-test/infra/vppinstance.go
@@ -33,19 +33,15 @@ const vppConfigTemplate = `unix {
   nodaemon
   log %[1]s%[4]s
   full-coredump
+  coredump-size unlimited
   cli-listen %[1]s%[2]s
   runtime-dir %[1]s/var/run
-  gid vpp
 }
 
 api-trace {
   on
 }
 
-api-segment {
-  gid vpp
-}
-
 socksvr {
   socket-name %[1]s%[3]s
 }
@@ -479,7 +475,7 @@ func (vpp *VppInstance) createTap(
 }
 
 func (vpp *VppInstance) saveLogs() {
-	logTarget := vpp.Container.getLogDirPath() + "vppinstance-" + vpp.Container.Name + ".log"
+	logTarget := vpp.getSuite().getLogDirPath() + "vppinstance-" + vpp.Container.Name + ".log"
 	logSource := vpp.Container.GetHostWorkDir() + defaultLogFilePath
 	cmd := exec.Command("cp", logSource, logTarget)
 	vpp.getSuite().Log(cmd.String())