aboutsummaryrefslogtreecommitdiffstats
path: root/extras/hs-test/infra/hst_suite.go
diff options
context:
space:
mode:
Diffstat (limited to 'extras/hs-test/infra/hst_suite.go')
-rw-r--r--extras/hs-test/infra/hst_suite.go385
1 files changed, 253 insertions, 132 deletions
diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go
index a6ba14676d0..435001afba7 100644
--- a/extras/hs-test/infra/hst_suite.go
+++ b/extras/hs-test/infra/hst_suite.go
@@ -2,22 +2,27 @@ package hst
import (
"bufio"
- "errors"
"flag"
"fmt"
"io"
"log"
+ "net/http"
+ "net/http/httputil"
"os"
"os/exec"
"path/filepath"
"runtime"
+ "strconv"
"strings"
"time"
+ "github.com/edwarnicke/exechelper"
+
+ containerTypes "github.com/docker/docker/api/types/container"
+ "github.com/docker/docker/client"
"github.com/onsi/gomega/gmeasure"
"gopkg.in/yaml.v3"
- "github.com/edwarnicke/exechelper"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@@ -33,6 +38,11 @@ var IsVppDebug = flag.Bool("debug", false, "attach gdb to vpp")
var NConfiguredCpus = flag.Int("cpus", 1, "number of CPUs assigned to vpp")
var VppSourceFileDir = flag.String("vppsrc", "", "vpp source file directory")
var IsDebugBuild = flag.Bool("debug_build", false, "some paths are different with debug build")
+var UseCpu0 = flag.Bool("cpu0", false, "use cpu0")
+var IsLeakCheck = flag.Bool("leak_check", false, "run leak-check tests")
+var ParallelTotal = flag.Lookup("ginkgo.parallel.total")
+var DryRun = flag.Bool("dryrun", false, "set up containers but don't run tests")
+var NumaAwareCpuAlloc bool
var SuiteTimeout time.Duration
type HstSuite struct {
@@ -45,11 +55,39 @@ type HstSuite struct {
TestIds map[string]string
CpuAllocator *CpuAllocatorT
CpuContexts []*CpuContext
- CpuPerVpp int
+ CpuCount int
Ppid string
ProcessIndex string
Logger *log.Logger
LogFile *os.File
+ Docker *client.Client
+}
+
+type colors struct {
+ grn string
+ pur string
+ rst string
+}
+
+var Colors = colors{
+ grn: "\033[32m",
+ pur: "\033[35m",
+ rst: "\033[0m",
+}
+
+// used for colorful ReportEntry
+type StringerStruct struct {
+ Label string
+}
+
+// ColorableString for ReportEntry to use
+func (s StringerStruct) ColorableString() string {
+ return fmt.Sprintf("{{red}}%s{{/}}", s.Label)
+}
+
+// non-colorable String() is used by go's string formatting support but ignored by ReportEntry
+func (s StringerStruct) String() string {
+ return s.Label
}
func getTestFilename() string {
@@ -57,9 +95,30 @@ func getTestFilename() string {
return filepath.Base(filename)
}
+func (s *HstSuite) getLogDirPath() string {
+ testId := s.GetTestId()
+ testName := s.GetCurrentTestName()
+ logDirPath := logDir + testName + "/" + testId + "/"
+
+ cmd := exec.Command("mkdir", "-p", logDirPath)
+ if err := cmd.Run(); err != nil {
+ Fail("mkdir error: " + fmt.Sprint(err))
+ }
+
+ return logDirPath
+}
+
+func (s *HstSuite) newDockerClient() {
+ var err error
+ s.Docker, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+ s.AssertNil(err)
+ s.Log("docker client created")
+}
+
func (s *HstSuite) SetupSuite() {
s.CreateLogger()
- s.Log("Suite Setup")
+ s.Log("[* SUITE SETUP]")
+ s.newDockerClient()
RegisterFailHandler(func(message string, callerSkip ...int) {
s.HstFail()
Fail(message, callerSkip...)
@@ -73,12 +132,15 @@ func (s *HstSuite) SetupSuite() {
if err != nil {
Fail("failed to init cpu allocator: " + fmt.Sprint(err))
}
- s.CpuPerVpp = *NConfiguredCpus
+ s.CpuCount = *NConfiguredCpus
}
func (s *HstSuite) AllocateCpus() []int {
- cpuCtx, err := s.CpuAllocator.Allocate(len(s.StartedContainers), s.CpuPerVpp)
- s.AssertNil(err)
+ cpuCtx, err := s.CpuAllocator.Allocate(len(s.StartedContainers), s.CpuCount)
+ // using Fail instead of AssertNil to make error message more readable
+ if err != nil {
+ Fail(fmt.Sprint(err))
+ }
s.AddCpuContext(cpuCtx)
return cpuCtx.cpus
}
@@ -89,18 +151,29 @@ func (s *HstSuite) AddCpuContext(cpuCtx *CpuContext) {
func (s *HstSuite) TearDownSuite() {
defer s.LogFile.Close()
- s.Log("Suite Teardown")
+ defer s.Docker.Close()
+ if *IsPersistent || *DryRun {
+ return
+ }
+ s.Log("[* SUITE TEARDOWN]")
s.UnconfigureNetworkTopology()
}
func (s *HstSuite) TearDownTest() {
- s.Log("Test Teardown")
- if *IsPersistent {
+ s.Log("[* TEST TEARDOWN]")
+ if *IsPersistent || *DryRun {
return
}
+ coreDump := s.WaitForCoreDump()
s.ResetContainers()
- s.RemoveVolumes()
- s.Ip4AddrAllocator.DeleteIpAddresses()
+
+ if s.Ip4AddrAllocator != nil {
+ s.Ip4AddrAllocator.DeleteIpAddresses()
+ }
+
+ if coreDump {
+ Fail("VPP crashed")
+ }
}
func (s *HstSuite) SkipIfUnconfiguring() {
@@ -110,21 +183,12 @@ func (s *HstSuite) SkipIfUnconfiguring() {
}
func (s *HstSuite) SetupTest() {
- s.Log("Test Setup")
+ s.Log("[* TEST SETUP]")
s.StartedContainers = s.StartedContainers[:0]
s.SkipIfUnconfiguring()
- s.SetupVolumes()
s.SetupContainers()
}
-func (s *HstSuite) SetupVolumes() {
- for _, volume := range s.Volumes {
- cmd := "docker volume create --name=" + volume
- s.Log(cmd)
- exechelper.Run(cmd)
- }
-}
-
func (s *HstSuite) SetupContainers() {
for _, container := range s.Containers {
if !container.IsOptional {
@@ -171,7 +235,7 @@ func (s *HstSuite) HstFail() {
out, err := container.log(20)
if err != nil {
s.Log("An error occured while obtaining '" + container.Name + "' container logs: " + fmt.Sprint(err))
- s.Log("The container might not be running - check logs in " + container.getLogDirPath())
+ s.Log("The container might not be running - check logs in " + s.getLogDirPath())
continue
}
s.Log("\nvvvvvvvvvvvvvvv " +
@@ -183,31 +247,67 @@ func (s *HstSuite) HstFail() {
}
func (s *HstSuite) AssertNil(object interface{}, msgAndArgs ...interface{}) {
- Expect(object).To(BeNil(), msgAndArgs...)
+ ExpectWithOffset(2, object).To(BeNil(), msgAndArgs...)
}
func (s *HstSuite) AssertNotNil(object interface{}, msgAndArgs ...interface{}) {
- Expect(object).ToNot(BeNil(), msgAndArgs...)
+ ExpectWithOffset(2, object).ToNot(BeNil(), msgAndArgs...)
}
func (s *HstSuite) AssertEqual(expected, actual interface{}, msgAndArgs ...interface{}) {
- Expect(actual).To(Equal(expected), msgAndArgs...)
+ ExpectWithOffset(2, actual).To(Equal(expected), msgAndArgs...)
}
func (s *HstSuite) AssertNotEqual(expected, actual interface{}, msgAndArgs ...interface{}) {
- Expect(actual).ToNot(Equal(expected), msgAndArgs...)
+ ExpectWithOffset(2, actual).ToNot(Equal(expected), msgAndArgs...)
}
func (s *HstSuite) AssertContains(testString, contains interface{}, msgAndArgs ...interface{}) {
- Expect(testString).To(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...)
+ ExpectWithOffset(2, testString).To(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...)
}
func (s *HstSuite) AssertNotContains(testString, contains interface{}, msgAndArgs ...interface{}) {
- Expect(testString).ToNot(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...)
+ ExpectWithOffset(2, testString).ToNot(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertEmpty(object interface{}, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, object).To(BeEmpty(), msgAndArgs...)
}
func (s *HstSuite) AssertNotEmpty(object interface{}, msgAndArgs ...interface{}) {
- Expect(object).ToNot(BeEmpty(), msgAndArgs...)
+ ExpectWithOffset(2, object).ToNot(BeEmpty(), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertMatchError(actual, expected error, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, actual).To(MatchError(expected), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertGreaterThan(actual, expected interface{}, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, actual).Should(BeNumerically(">=", expected), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertTimeEqualWithinThreshold(actual, expected time.Time, threshold time.Duration, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, actual).Should(BeTemporally("~", expected, threshold), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertHttpStatus(resp *http.Response, expectedStatus int, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, resp).To(HaveHTTPStatus(expectedStatus), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertHttpHeaderWithValue(resp *http.Response, key string, value interface{}, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, resp).To(HaveHTTPHeaderWithValue(key, value), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertHttpHeaderNotPresent(resp *http.Response, key string, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, resp.Header.Get(key)).To(BeEmpty(), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertHttpContentLength(resp *http.Response, expectedContentLen int64, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, resp).To(HaveHTTPHeaderWithValue("Content-Length", strconv.FormatInt(expectedContentLen, 10)), msgAndArgs...)
+}
+
+func (s *HstSuite) AssertHttpBody(resp *http.Response, expectedBody string, msgAndArgs ...interface{}) {
+ ExpectWithOffset(2, resp).To(HaveHTTPBody(expectedBody), msgAndArgs...)
}
func (s *HstSuite) CreateLogger() {
@@ -222,13 +322,13 @@ func (s *HstSuite) CreateLogger() {
// Logs to files by default, logs to stdout when VERBOSE=true with GinkgoWriter
// to keep console tidy
-func (s *HstSuite) Log(arg any) {
- logs := strings.Split(fmt.Sprint(arg), "\n")
+func (s *HstSuite) Log(log any, arg ...any) {
+ logs := strings.Split(fmt.Sprintf(fmt.Sprint(log), arg...), "\n")
for _, line := range logs {
s.Logger.Println(line)
}
if *IsVerbose {
- GinkgoWriter.Println(arg)
+ GinkgoWriter.Println(fmt.Sprintf(fmt.Sprint(log), arg...))
}
}
@@ -242,32 +342,103 @@ func (s *HstSuite) SkipIfMultiWorker(args ...any) {
}
}
-func (s *HstSuite) SkipUnlessExtendedTestsBuilt() {
- imageName := "hs-test/nginx-http3"
+func (s *HstSuite) SkipIfNotEnoughAvailableCpus() {
+ var maxRequestedCpu int
+ availableCpus := len(s.CpuAllocator.cpus) - 1
+
+ if *UseCpu0 {
+ availableCpus++
+ }
+
+ if s.CpuAllocator.runningInCi {
+ maxRequestedCpu = ((s.CpuAllocator.buildNumber + 1) * s.CpuAllocator.maxContainerCount * s.CpuCount)
+ } else {
+ maxRequestedCpu = (GinkgoParallelProcess() * s.CpuAllocator.maxContainerCount * s.CpuCount)
+ }
+
+ if availableCpus < maxRequestedCpu {
+ s.Skip(fmt.Sprintf("Test case cannot allocate requested cpus "+
+ "(%d cpus * %d containers, %d available). Try using 'CPU0=true'",
+ s.CpuCount, s.CpuAllocator.maxContainerCount, availableCpus))
+ }
+}
+
+func (s *HstSuite) SkipUnlessLeakCheck() {
+ if !*IsLeakCheck {
+ s.Skip("leak-check tests excluded")
+ }
+}
- cmd := exec.Command("docker", "images", imageName)
- byteOutput, err := cmd.CombinedOutput()
+func (s *HstSuite) WaitForCoreDump() bool {
+ var filename string
+ dir, err := os.Open(s.getLogDirPath())
if err != nil {
- s.Log("error while searching for docker image")
- return
+ s.Log(err)
+ return false
+ }
+ defer dir.Close()
+
+ files, err := dir.Readdirnames(0)
+ if err != nil {
+ s.Log(err)
+ return false
+ }
+ for _, file := range files {
+ if strings.Contains(file, "core") {
+ filename = file
+ }
}
- if !strings.Contains(string(byteOutput), imageName) {
- s.Skip("extended tests not built")
+ timeout := 60
+ waitTime := 5
+
+ if filename != "" {
+ corePath := s.getLogDirPath() + filename
+ s.Log(fmt.Sprintf("WAITING FOR CORE DUMP (%s)", corePath))
+ for i := waitTime; i <= timeout; i += waitTime {
+ fileInfo, err := os.Stat(corePath)
+ if err != nil {
+ s.Log("Error while reading file info: " + fmt.Sprint(err))
+ return true
+ }
+ currSize := fileInfo.Size()
+ s.Log(fmt.Sprintf("Waiting %ds/%ds...", i, timeout))
+ time.Sleep(time.Duration(waitTime) * time.Second)
+ fileInfo, _ = os.Stat(corePath)
+
+ if currSize == fileInfo.Size() {
+ debug := ""
+ if *IsDebugBuild {
+ debug = "_debug"
+ }
+ vppBinPath := fmt.Sprintf("../../build-root/build-vpp%s-native/vpp/bin/vpp", debug)
+ pluginsLibPath := fmt.Sprintf("build-root/build-vpp%s-native/vpp/lib/x86_64-linux-gnu/vpp_plugins", debug)
+ cmd := fmt.Sprintf("sudo gdb %s -c %s -ex 'set solib-search-path %s/%s' -ex 'bt full' -batch", vppBinPath, corePath, *VppSourceFileDir, pluginsLibPath)
+ s.Log(cmd)
+ output, _ := exechelper.Output(cmd)
+ AddReportEntry("VPP Backtrace", StringerStruct{Label: string(output)})
+ os.WriteFile(s.getLogDirPath()+"backtrace.log", output, os.FileMode(0644))
+ if s.CpuAllocator.runningInCi {
+ err = os.Remove(corePath)
+ if err == nil {
+ s.Log("removed " + corePath)
+ } else {
+ s.Log(err)
+ }
+ }
+ return true
+ }
+ }
}
+ return false
}
func (s *HstSuite) ResetContainers() {
for _, container := range s.StartedContainers {
container.stop()
- exechelper.Run("docker rm " + container.Name)
- }
-}
-
-func (s *HstSuite) RemoveVolumes() {
- for _, volumeName := range s.Volumes {
- cmd := "docker volume rm " + volumeName
- exechelper.Run(cmd)
- os.RemoveAll(volumeName)
+ s.Log("Removing container " + container.Name)
+ if err := s.Docker.ContainerRemove(container.ctx, container.ID, containerTypes.RemoveOptions{RemoveVolumes: true}); err != nil {
+ s.Log(err)
+ }
}
}
@@ -322,6 +493,13 @@ func (s *HstSuite) LoadContainerTopology(topologyName string) {
}
s.Containers[newContainer.Name] = newContainer
}
+
+ if *DryRun {
+ s.Log(Colors.pur + "* Containers used by this suite (some might already be running):" + Colors.rst)
+ for name := range s.Containers {
+ s.Log("%sdocker start %s && docker exec -it %s bash%s", Colors.pur, name, name, Colors.rst)
+ }
+ }
}
func (s *HstSuite) LoadNetworkTopology(topologyName string) {
@@ -409,14 +587,18 @@ func (s *HstSuite) ConfigureNetworkTopology(topologyName string) {
}
func (s *HstSuite) UnconfigureNetworkTopology() {
- if *IsPersistent {
- return
- }
for _, nc := range s.NetConfigs {
nc.unconfigure()
}
}
+func (s *HstSuite) LogStartedContainers() {
+ s.Log("%s* Started containers:%s", Colors.grn, Colors.rst)
+ for _, container := range s.StartedContainers {
+ s.Log(Colors.grn + container.Name + Colors.rst)
+ }
+}
+
func (s *HstSuite) GetTestId() string {
testName := s.GetCurrentTestName()
@@ -448,89 +630,12 @@ func (s *HstSuite) GetPortFromPpid() string {
return port[len(port)-3:] + s.ProcessIndex
}
-func (s *HstSuite) StartServerApp(running chan error, done chan struct{}, env []string) {
- cmd := exec.Command("iperf3", "-4", "-s", "-p", s.GetPortFromPpid())
- if env != nil {
- cmd.Env = env
- }
- s.Log(cmd)
- err := cmd.Start()
- if err != nil {
- msg := fmt.Errorf("failed to start iperf server: %v", err)
- running <- msg
- return
- }
- running <- nil
- <-done
- cmd.Process.Kill()
-}
-
-func (s *HstSuite) StartClientApp(ipAddress string, env []string, clnCh chan error, clnRes chan string) {
- defer func() {
- clnCh <- nil
- }()
-
- nTries := 0
-
- for {
- cmd := exec.Command("iperf3", "-c", ipAddress, "-u", "-l", "1460", "-b", "10g", "-p", s.GetPortFromPpid())
- if env != nil {
- cmd.Env = env
- }
- s.Log(cmd)
- o, err := cmd.CombinedOutput()
- if err != nil {
- if nTries > 5 {
- clnCh <- fmt.Errorf("failed to start client app '%s'.\n%s", err, o)
- return
- }
- time.Sleep(1 * time.Second)
- nTries++
- continue
- } else {
- clnRes <- fmt.Sprintf("Client output: %s", o)
- }
- break
- }
-}
-
-func (s *HstSuite) StartHttpServer(running chan struct{}, done chan struct{}, addressPort, netNs string) {
- cmd := newCommand([]string{"./http_server", addressPort, s.Ppid, s.ProcessIndex}, netNs)
- err := cmd.Start()
- s.Log(cmd)
- if err != nil {
- s.Log("Failed to start http server: " + fmt.Sprint(err))
- return
- }
- running <- struct{}{}
- <-done
- cmd.Process.Kill()
-}
-
-func (s *HstSuite) StartWget(finished chan error, server_ip, port, query, netNs string) {
- defer func() {
- finished <- errors.New("wget error")
- }()
-
- cmd := newCommand([]string{"wget", "--timeout=10", "--no-proxy", "--tries=5", "-O", "/dev/null", server_ip + ":" + port + "/" + query},
- netNs)
- s.Log(cmd)
- o, err := cmd.CombinedOutput()
- if err != nil {
- finished <- fmt.Errorf("wget error: '%v\n\n%s'", err, o)
- return
- } else if !strings.Contains(string(o), "200 OK") {
- finished <- fmt.Errorf("wget error: response not 200 OK")
- return
- }
- finished <- nil
-}
-
/*
-runBenchmark creates Gomega's experiment with the passed-in name and samples the passed-in callback repeatedly (samplesNum times),
+RunBenchmark creates Gomega's experiment with the passed-in name and samples the passed-in callback repeatedly (samplesNum times),
passing in suite context, experiment and your data.
You can also instruct runBenchmark to run with multiple concurrent workers.
+Note that if running in parallel Gomega returns from Sample when spins up all samples and does not wait until all finished.
You can record multiple named measurements (float64 or duration) within passed-in callback.
runBenchmark then produces report to show statistical distribution of measurements.
*/
@@ -543,3 +648,19 @@ func (s *HstSuite) RunBenchmark(name string, samplesNum, parallelNum int, callba
}, gmeasure.SamplingConfig{N: samplesNum, NumParallel: parallelNum})
AddReportEntry(experiment.Name, experiment)
}
+
+/*
+LogHttpReq is Gomega's ghttp server handler which logs received HTTP request.
+
+You should put it at the first place, so request is logged always.
+*/
+func (s *HstSuite) LogHttpReq(body bool) http.HandlerFunc {
+ return func(w http.ResponseWriter, req *http.Request) {
+ dump, err := httputil.DumpRequest(req, body)
+ if err == nil {
+ s.Log("\n> Received request (" + req.RemoteAddr + "):\n" +
+ string(dump) +
+ "\n------------------------------\n")
+ }
+ }
+}