From b9464cde7d3288f8feeeaf945ce933b276f07d82 Mon Sep 17 00:00:00 2001 From: Adrian Villin Date: Mon, 27 May 2024 09:52:59 -0400 Subject: hs-test: pin CPUs to containers Type: test Change-Id: I412be2dec7ff352740e50e838e0ac466bf0a6674 Signed-off-by: Adrian Villin --- extras/hs-test/Makefile | 2 +- extras/hs-test/container.go | 13 +++++++++++-- extras/hs-test/cpu.go | 26 +++++++++++++++----------- extras/hs-test/hst_suite.go | 30 +++++++++++++++--------------- extras/hs-test/suite_nginx_test.go | 3 +-- extras/hs-test/suite_no_topo_test.go | 3 +-- extras/hs-test/suite_ns_test.go | 3 +-- extras/hs-test/suite_veth_test.go | 6 ++---- 8 files changed, 47 insertions(+), 39 deletions(-) (limited to 'extras/hs-test') diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile index e247bf44160..0db3958604e 100644 --- a/extras/hs-test/Makefile +++ b/extras/hs-test/Makefile @@ -68,7 +68,7 @@ help: @echo " UNCONFIGURE=[true|false] - unconfigure selected test" @echo " DEBUG=[true|false] - attach VPP to GDB" @echo " TEST=[test-name] - specific test to run" - @echo " CPUS=[n-cpus] - number of cpus to run with vpp" + @echo " CPUS=[n-cpus] - number of cpus to allocate to VPP and containers" @echo " VPPSRC=[path-to-vpp-src] - path to vpp source files (for gdb)" @echo " PARALLEL=[n-cpus] - number of test processes to spawn to run in parallel" @echo " REPEAT=[n] - repeat tests up to N times or until a failure occurs" diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go index 0bdc3a24996..83693f47d78 100644 --- a/extras/hs-test/container.go +++ b/extras/hs-test/container.go @@ -37,6 +37,7 @@ type Container struct { volumes map[string]Volume envVars map[string]string vppInstance *VppInstance + allocatedCpus []int } func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error) { @@ -160,6 +161,12 @@ func (c *Container) create() error { return exechelper.Run(cmd) } +func (c *Container) allocateCpus() { + c.suite.containerCount += 1 + c.allocatedCpus = c.suite.AllocateCpus() + c.suite.log("Allocated CPUs " + fmt.Sprint(c.allocatedCpus) + " to container " + c.name) +} + func (c *Container) start() error { cmd := "docker start " + c.name c.suite.log(cmd) @@ -175,6 +182,9 @@ func (c *Container) prepareCommand() (string, error) { if c.runDetached { cmd += " -d" } + + c.allocateCpus() + cmd += fmt.Sprintf(" --cpuset-cpus=\"%d-%d\"", c.allocatedCpus[0], c.allocatedCpus[len(c.allocatedCpus)-1]) cmd += " " + c.getContainerArguments() c.suite.log(cmd) @@ -239,7 +249,6 @@ func (c *Container) newVppInstance(cpus []int, additionalConfigs ...Stanza) (*Vp vpp := new(VppInstance) vpp.container = c vpp.cpus = cpus - c.suite.vppContainerCount += 1 vpp.additionalConfig = append(vpp.additionalConfig, additionalConfigs...) c.vppInstance = vpp return vpp, nil @@ -287,7 +296,7 @@ func (c *Container) exec(command string, arguments ...any) string { GinkgoHelper() c.suite.log(containerExecCommand) byteOutput, err := exechelper.CombinedOutput(containerExecCommand) - c.suite.assertNil(err, err) + c.suite.assertNil(err, fmt.Sprint(err)) return string(byteOutput) } diff --git a/extras/hs-test/cpu.go b/extras/hs-test/cpu.go index 69b4cabd4e3..49a7dfb02f8 100644 --- a/extras/hs-test/cpu.go +++ b/extras/hs-test/cpu.go @@ -23,22 +23,26 @@ type CpuAllocatorT struct { var cpuAllocator *CpuAllocatorT = nil -func (c *CpuAllocatorT) Allocate(vppContainerCount int, nCpus int) (*CpuContext, error) { +func (c *CpuAllocatorT) Allocate(containerCount int, nCpus int) (*CpuContext, error) { var cpuCtx CpuContext - maxCpu := GinkgoParallelProcess() * 2 * nCpus - minCpu := (GinkgoParallelProcess() - 1) * 2 * nCpus - if len(c.cpus) < maxCpu { - vppContainerCount += 1 + + // splitting cpus into equal parts; this will over-allocate cores but it's good enough for now + maxContainerCount := 4 + // skip CPU 0 + minCpu := ((GinkgoParallelProcess() - 1) * maxContainerCount * nCpus) + 1 + maxCpu := (GinkgoParallelProcess() * maxContainerCount * nCpus) + + if len(c.cpus)-1 < maxCpu { err := fmt.Errorf("could not allocate %d CPUs; available: %d; attempted to allocate cores %d-%d", - nCpus*vppContainerCount, len(c.cpus), minCpu, minCpu+nCpus*vppContainerCount) + nCpus*containerCount, len(c.cpus)-1, minCpu, maxCpu) return nil, err } - if vppContainerCount == 0 { - cpuCtx.cpus = c.cpus[minCpu : maxCpu-nCpus] - } else if vppContainerCount == 1 { - cpuCtx.cpus = c.cpus[minCpu+nCpus : maxCpu] + if containerCount == 1 { + cpuCtx.cpus = c.cpus[minCpu : minCpu+nCpus] + } else if containerCount > 1 && containerCount <= maxContainerCount { + cpuCtx.cpus = c.cpus[minCpu+(nCpus*(containerCount-1)) : minCpu+(nCpus*containerCount)] } else { - return nil, fmt.Errorf("too many VPP containers; CPU allocation for >2 VPP containers is not implemented yet") + return nil, fmt.Errorf("too many containers; CPU allocation for >%d containers is not implemented", maxContainerCount) } cpuCtx.cpuAllocator = c diff --git a/extras/hs-test/hst_suite.go b/extras/hs-test/hst_suite.go index 9cb79c563fa..b8c0a424d58 100644 --- a/extras/hs-test/hst_suite.go +++ b/extras/hs-test/hst_suite.go @@ -30,19 +30,19 @@ var nConfiguredCpus = flag.Int("cpus", 1, "number of CPUs assigned to vpp") var vppSourceFileDir = flag.String("vppsrc", "", "vpp source file directory") type HstSuite struct { - containers map[string]*Container - vppContainerCount int - volumes []string - netConfigs []NetConfig - netInterfaces map[string]*NetInterface - ip4AddrAllocator *Ip4AddressAllocator - testIds map[string]string - cpuAllocator *CpuAllocatorT - cpuContexts []*CpuContext - cpuPerVpp int - pid string - logger *log.Logger - logFile *os.File + containers map[string]*Container + containerCount int + volumes []string + netConfigs []NetConfig + netInterfaces map[string]*NetInterface + ip4AddrAllocator *Ip4AddressAllocator + testIds map[string]string + cpuAllocator *CpuAllocatorT + cpuContexts []*CpuContext + cpuPerVpp int + pid string + logger *log.Logger + logFile *os.File } func (s *HstSuite) SetupSuite() { @@ -62,7 +62,7 @@ func (s *HstSuite) SetupSuite() { } func (s *HstSuite) AllocateCpus() []int { - cpuCtx, err := s.cpuAllocator.Allocate(s.vppContainerCount, s.cpuPerVpp) + cpuCtx, err := s.cpuAllocator.Allocate(s.containerCount, s.cpuPerVpp) s.assertNil(err) s.AddCpuContext(cpuCtx) return cpuCtx.cpus @@ -96,7 +96,7 @@ func (s *HstSuite) skipIfUnconfiguring() { func (s *HstSuite) SetupTest() { s.log("Test Setup") - s.vppContainerCount = 0 + s.containerCount = 0 s.skipIfUnconfiguring() s.setupVolumes() s.setupContainers() diff --git a/extras/hs-test/suite_nginx_test.go b/extras/hs-test/suite_nginx_test.go index c559496e71b..ef4ca5c4ae4 100644 --- a/extras/hs-test/suite_nginx_test.go +++ b/extras/hs-test/suite_nginx_test.go @@ -48,10 +48,9 @@ func (s *NginxSuite) SetupTest() { append("enable"). append("use-app-socket-api").close() - cpus := s.AllocateCpus() // ... for proxy vppProxyContainer := s.getContainerByName(vppProxyContainerName) - proxyVpp, _ := vppProxyContainer.newVppInstance(cpus, sessionConfig) + proxyVpp, _ := vppProxyContainer.newVppInstance(vppProxyContainer.allocatedCpus, sessionConfig) s.assertNil(proxyVpp.start()) clientInterface := s.getInterfaceByName(mirroringClientInterfaceName) diff --git a/extras/hs-test/suite_no_topo_test.go b/extras/hs-test/suite_no_topo_test.go index 625dca9f3cf..19a3e1fd288 100644 --- a/extras/hs-test/suite_no_topo_test.go +++ b/extras/hs-test/suite_no_topo_test.go @@ -45,9 +45,8 @@ func (s *NoTopoSuite) SetupTest() { append("enable"). append("use-app-socket-api").close() - cpus := s.AllocateCpus() container := s.getContainerByName(singleTopoContainerVpp) - vpp, _ := container.newVppInstance(cpus, sessionConfig) + vpp, _ := container.newVppInstance(container.allocatedCpus, sessionConfig) s.assertNil(vpp.start()) tapInterface := s.getInterfaceByName(tapInterfaceName) diff --git a/extras/hs-test/suite_ns_test.go b/extras/hs-test/suite_ns_test.go index 85b90911c2f..377046cdaeb 100644 --- a/extras/hs-test/suite_ns_test.go +++ b/extras/hs-test/suite_ns_test.go @@ -48,9 +48,8 @@ func (s *NsSuite) SetupTest() { append("evt_qs_memfd_seg"). append("event-queue-length 100000").close() - cpus := s.AllocateCpus() container := s.getContainerByName("vpp") - vpp, _ := container.newVppInstance(cpus, sessionConfig) + vpp, _ := container.newVppInstance(container.allocatedCpus, sessionConfig) s.assertNil(vpp.start()) idx, err := vpp.createAfPacket(s.getInterfaceByName(serverInterface)) diff --git a/extras/hs-test/suite_veth_test.go b/extras/hs-test/suite_veth_test.go index d47bf8c52a9..c3038ca3727 100644 --- a/extras/hs-test/suite_veth_test.go +++ b/extras/hs-test/suite_veth_test.go @@ -50,8 +50,7 @@ func (s *VethsSuite) SetupTest() { // ... For server serverContainer := s.getContainerByName("server-vpp") - cpus := s.AllocateCpus() - serverVpp, err := serverContainer.newVppInstance(cpus, sessionConfig) + serverVpp, err := serverContainer.newVppInstance(serverContainer.allocatedCpus, sessionConfig) s.assertNotNil(serverVpp, fmt.Sprint(err)) s.setupServerVpp() @@ -59,8 +58,7 @@ func (s *VethsSuite) SetupTest() { // ... For client clientContainer := s.getContainerByName("client-vpp") - cpus = s.AllocateCpus() - clientVpp, err := clientContainer.newVppInstance(cpus, sessionConfig) + clientVpp, err := clientContainer.newVppInstance(clientContainer.allocatedCpus, sessionConfig) s.assertNotNil(clientVpp, fmt.Sprint(err)) s.setupClientVpp() -- cgit 1.2.3-korg