diff options
Diffstat (limited to 'extras/hs-test/infra')
| -rw-r--r-- | extras/hs-test/infra/container.go | 205 | ||||
| -rw-r--r-- | extras/hs-test/infra/hst_suite.go | 40 | ||||
| -rw-r--r-- | extras/hs-test/infra/suite_nginx.go | 2 |
3 files changed, 186 insertions, 61 deletions
diff --git a/extras/hs-test/infra/container.go b/extras/hs-test/infra/container.go index 3e8ccb4550f..d71761b2920 100644 --- a/extras/hs-test/infra/container.go +++ b/extras/hs-test/infra/container.go @@ -1,13 +1,21 @@ package hst import ( + "bytes" + "context" "fmt" "os" "os/exec" + "slices" + "strconv" "strings" "text/template" "time" + containerTypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-units" "github.com/edwarnicke/exechelper" . "github.com/onsi/ginkgo/v2" ) @@ -32,12 +40,14 @@ type Container struct { IsOptional bool RunDetached bool Name string + ID string Image string ExtraRunningArgs string Volumes map[string]Volume EnvVars map[string]string VppInstance *VppInstance AllocatedCpus []int + ctx context.Context } func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error) { @@ -52,6 +62,7 @@ func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error container.EnvVars = make(map[string]string) container.Name = containerName container.Suite = suite + container.ctx = context.Background() if Image, ok := yamlInput["image"]; ok { container.Image = Image.(string) @@ -133,8 +144,6 @@ func (c *Container) GetContainerWorkDir() (res string) { func (c *Container) getContainerArguments() string { args := "--ulimit nofile=90000:90000 --cap-add=all --privileged --network host" - c.allocateCpus() - args += fmt.Sprintf(" --cpuset-cpus=\"%d-%d\"", c.AllocatedCpus[0], c.AllocatedCpus[len(c.AllocatedCpus)-1]) args += c.getVolumesAsCliOption() args += c.getEnvVarsAsCliOption() if *VppSourceFileDir != "" { @@ -145,22 +154,57 @@ func (c *Container) getContainerArguments() string { return args } -func (c *Container) runWithRetry(cmd string) error { - nTries := 5 - for i := 0; i < nTries; i++ { - err := exechelper.Run(cmd) - if err == nil { - return nil - } - time.Sleep(1 * time.Second) - } - return fmt.Errorf("failed to run container command") +func (c *Container) PullDockerImage(name string, ctx context.Context) { + // "func (*Client) ImagePull" doesn't work, returns "No such image" + c.Suite.Log("Pulling image: " + name) + _, err := exechelper.CombinedOutput("docker pull " + name) + c.Suite.AssertNil(err) } +// Creates a container func (c *Container) Create() error { - cmd := "docker create " + c.getContainerArguments() - c.Suite.Log(cmd) - return exechelper.Run(cmd) + var sliceOfImageNames []string + images, err := c.Suite.Docker.ImageList(c.ctx, image.ListOptions{}) + c.Suite.AssertNil(err) + + for _, image := range images { + sliceOfImageNames = append(sliceOfImageNames, strings.Split(image.RepoTags[0], ":")[0]) + } + if !slices.Contains(sliceOfImageNames, c.Image) { + c.PullDockerImage(c.Image, c.ctx) + } + + c.allocateCpus() + cpuSet := fmt.Sprintf("%d-%d", c.AllocatedCpus[0], c.AllocatedCpus[len(c.AllocatedCpus)-1]) + resp, err := c.Suite.Docker.ContainerCreate( + c.ctx, + &containerTypes.Config{ + Image: c.Image, + Env: c.getEnvVars(), + Cmd: strings.Split(c.ExtraRunningArgs, " "), + }, + &containerTypes.HostConfig{ + Resources: containerTypes.Resources{ + Ulimits: []*units.Ulimit{ + { + Name: "nofile", + Soft: 90000, + Hard: 90000, + }, + }, + CpusetCpus: cpuSet, + }, + CapAdd: []string{"ALL"}, + Privileged: true, + NetworkMode: "host", + Binds: c.getVolumesAsSlice(), + }, + nil, + nil, + c.Name, + ) + c.ID = resp.ID + return err } func (c *Container) allocateCpus() { @@ -169,10 +213,45 @@ func (c *Container) allocateCpus() { c.Suite.Log("Allocated CPUs " + fmt.Sprint(c.AllocatedCpus) + " to container " + c.Name) } +// Starts a container func (c *Container) Start() error { - cmd := "docker start " + c.Name - c.Suite.Log(cmd) - return c.runWithRetry(cmd) + var err error + for nTries := 0; nTries < 5; nTries++ { + err = c.Suite.Docker.ContainerStart(c.ctx, c.ID, containerTypes.StartOptions{}) + if err == nil { + continue + } + c.Suite.Log("Error while starting " + c.Name + ". Retrying...") + time.Sleep(1 * time.Second) + } + + return err +} + +func (c *Container) GetOutput() (string, string) { + // Wait for the container to finish executing + statusCh, errCh := c.Suite.Docker.ContainerWait(c.ctx, c.ID, containerTypes.WaitConditionNotRunning) + select { + case err := <-errCh: + c.Suite.AssertNil(err) + case <-statusCh: + } + + // Get the logs from the container + logOptions := containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true} + logReader, err := c.Suite.Docker.ContainerLogs(c.ctx, c.ID, logOptions) + c.Suite.AssertNil(err) + defer logReader.Close() + + var stdoutBuf, stderrBuf bytes.Buffer + + // Use stdcopy.StdCopy to demultiplex the multiplexed stream + _, err = stdcopy.StdCopy(&stdoutBuf, &stderrBuf, logReader) + c.Suite.AssertNil(err) + + stdout := stdoutBuf.String() + stderr := stderrBuf.String() + return stdout, stderr } func (c *Container) prepareCommand() (string, error) { @@ -180,7 +259,7 @@ func (c *Container) prepareCommand() (string, error) { return "", fmt.Errorf("run container failed: name is blank") } - cmd := "docker run " + cmd := "docker exec " if c.RunDetached { cmd += " -d" } @@ -201,12 +280,10 @@ func (c *Container) CombinedOutput() (string, error) { return string(byteOutput), err } -func (c *Container) Run() error { - cmd, err := c.prepareCommand() - if err != nil { - return err - } - return c.runWithRetry(cmd) +// Creates and starts a container +func (c *Container) Run() { + c.Suite.AssertNil(c.Create()) + c.Suite.AssertNil(c.Start()) } func (c *Container) addVolume(hostDir string, containerDir string, isDefaultWorkDir bool) { @@ -217,6 +294,22 @@ func (c *Container) addVolume(hostDir string, containerDir string, isDefaultWork c.Volumes[hostDir] = volume } +func (c *Container) getVolumesAsSlice() []string { + var volumeSlice []string + + if *VppSourceFileDir != "" { + volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", *VppSourceFileDir, *VppSourceFileDir)) + } + + if len(c.Volumes) > 0 { + for _, volume := range c.Volumes { + volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", volume.HostDir, volume.ContainerDir)) + } + } + + return volumeSlice +} + func (c *Container) getVolumesAsCliOption() string { cliOption := "" @@ -245,6 +338,18 @@ func (c *Container) getEnvVarsAsCliOption() string { return cliOption } +func (c *Container) getEnvVars() []string { + var envVars []string + if len(c.EnvVars) == 0 { + return envVars + } + + for name, value := range c.EnvVars { + envVars = append(envVars, fmt.Sprintf("%s=%s", name, value)) + } + return envVars +} + func (c *Container) newVppInstance(cpus []int, additionalConfigs ...Stanza) (*VppInstance, error) { vpp := new(VppInstance) vpp.Container = c @@ -317,33 +422,48 @@ func (c *Container) getLogDirPath() string { func (c *Container) saveLogs() { testLogFilePath := c.getLogDirPath() + "container-" + c.Name + ".log" - cmd := exec.Command("docker", "logs", "--details", "-t", c.Name) - c.Suite.Log(cmd) - output, err := cmd.CombinedOutput() + logs, err := c.log(0) if err != nil { c.Suite.Log(err) + return } f, err := os.Create(testLogFilePath) if err != nil { - Fail("file create error: " + fmt.Sprint(err)) + c.Suite.Log(err) + return } - fmt.Fprint(f, string(output)) - f.Close() + defer f.Close() + fmt.Fprint(f, logs) } -// Outputs logs from docker containers. Set 'maxLines' to 0 to output the full log. +// Returns logs from docker containers. Set 'maxLines' to 0 to output the full log. func (c *Container) log(maxLines int) (string, error) { - var cmd string + var logOptions containerTypes.LogsOptions if maxLines == 0 { - cmd = "docker logs " + c.Name + logOptions = containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true, Details: true} } else { - cmd = fmt.Sprintf("docker logs --tail %d %s", maxLines, c.Name) + logOptions = containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true, Details: true, Tail: strconv.Itoa(maxLines)} } - c.Suite.Log(cmd) - o, err := exechelper.CombinedOutput(cmd) - return string(o), err + out, err := c.Suite.Docker.ContainerLogs(c.ctx, c.ID, logOptions) + if err != nil { + c.Suite.Log(err) + return "", err + } + defer out.Close() + + var stdoutBuf, stderrBuf bytes.Buffer + + _, err = stdcopy.StdCopy(&stdoutBuf, &stderrBuf, out) + if err != nil { + c.Suite.Log(err) + } + + stdout := stdoutBuf.String() + stderr := stderrBuf.String() + + return stdout + " " + stderr, err } func (c *Container) stop() error { @@ -353,8 +473,13 @@ func (c *Container) stop() error { } c.VppInstance = nil c.saveLogs() - c.Suite.Log("docker stop " + c.Name + " -t 0") - return exechelper.Run("docker stop " + c.Name + " -t 0") + + c.Suite.Log("Stopping container " + c.Name) + timeout := 0 + if err := c.Suite.Docker.ContainerStop(c.ctx, c.ID, containerTypes.StopOptions{Timeout: &timeout}); err != nil { + return err + } + return nil } func (c *Container) CreateConfig(targetConfigName string, templateName string, values any) { diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go index ac255b9fd44..975e01d5b8e 100644 --- a/extras/hs-test/infra/hst_suite.go +++ b/extras/hs-test/infra/hst_suite.go @@ -14,10 +14,11 @@ import ( "strings" "time" + containerTypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" "github.com/onsi/gomega/gmeasure" "gopkg.in/yaml.v3" - "github.com/edwarnicke/exechelper" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -52,6 +53,7 @@ type HstSuite struct { ProcessIndex string Logger *log.Logger LogFile *os.File + Docker *client.Client } func getTestFilename() string { @@ -59,8 +61,16 @@ func getTestFilename() string { return filepath.Base(filename) } +func (s *HstSuite) newDockerClient() { + var err error + s.Docker, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + s.AssertNil(err) + s.Log("docker client created") +} + func (s *HstSuite) SetupSuite() { s.CreateLogger() + s.newDockerClient() s.Log("Suite Setup") RegisterFailHandler(func(message string, callerSkip ...int) { s.HstFail() @@ -94,6 +104,7 @@ func (s *HstSuite) AddCpuContext(cpuCtx *CpuContext) { func (s *HstSuite) TearDownSuite() { defer s.LogFile.Close() + defer s.Docker.Close() s.Log("Suite Teardown") s.UnconfigureNetworkTopology() } @@ -104,7 +115,6 @@ func (s *HstSuite) TearDownTest() { return } s.ResetContainers() - s.RemoveVolumes() if s.Ip4AddrAllocator != nil { s.Ip4AddrAllocator.DeleteIpAddresses() @@ -121,18 +131,9 @@ func (s *HstSuite) SetupTest() { s.Log("Test Setup") s.StartedContainers = s.StartedContainers[:0] s.SkipIfUnconfiguring() - s.SetupVolumes() s.SetupContainers() } -func (s *HstSuite) SetupVolumes() { - for _, volume := range s.Volumes { - cmd := "docker volume create --name=" + volume - s.Log(cmd) - exechelper.Run(cmd) - } -} - func (s *HstSuite) SetupContainers() { for _, container := range s.Containers { if !container.IsOptional { @@ -214,6 +215,10 @@ func (s *HstSuite) AssertNotContains(testString, contains interface{}, msgAndArg Expect(testString).ToNot(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...) } +func (s *HstSuite) AssertEmpty(object interface{}, msgAndArgs ...interface{}) { + Expect(object).To(BeEmpty(), msgAndArgs...) +} + func (s *HstSuite) AssertNotEmpty(object interface{}, msgAndArgs ...interface{}) { Expect(object).ToNot(BeEmpty(), msgAndArgs...) } @@ -283,15 +288,10 @@ func (s *HstSuite) SkipUnlessExtendedTestsBuilt() { func (s *HstSuite) ResetContainers() { for _, container := range s.StartedContainers { container.stop() - exechelper.Run("docker rm " + container.Name) - } -} - -func (s *HstSuite) RemoveVolumes() { - for _, volumeName := range s.Volumes { - cmd := "docker volume rm " + volumeName - exechelper.Run(cmd) - os.RemoveAll(volumeName) + s.Log("Removing container " + container.Name) + if err := s.Docker.ContainerRemove(container.ctx, container.ID, containerTypes.RemoveOptions{RemoveVolumes: true}); err != nil { + s.Log(err) + } } } diff --git a/extras/hs-test/infra/suite_nginx.go b/extras/hs-test/infra/suite_nginx.go index bb1bdb0f42b..3a8b28e52eb 100644 --- a/extras/hs-test/infra/suite_nginx.go +++ b/extras/hs-test/infra/suite_nginx.go @@ -66,7 +66,7 @@ func (s *NginxSuite) SetupTest() { s.AssertNil(proxyVpp.createTap(serverInterface, 2)) nginxContainer := s.GetTransientContainerByName(NginxProxyContainerName) - nginxContainer.Create() + s.AssertNil(nginxContainer.Create()) values := struct { Proxy string |