hs-test: use docker sdk

Type: test

Change-Id: I9d6b15ca6a9aac3343e10f480dec43c4c538f1b7
Signed-off-by: Adrian Villin <avillin@cisco.com>

3 files changed, 186 insertions, 61 deletions

diff --git a/extras/hs-test/infra/container.go b/extras/hs-test/infra/container.go
index 3e8ccb4550f..d71761b2920 100644
--- a/extras/hs-test/infra/container.go
+++ b/extras/hs-test/infra/container.go
@@ -1,13 +1,21 @@
 package hst
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
+	"slices"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
 
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/docker/go-units"
 	"github.com/edwarnicke/exechelper"
 	. "github.com/onsi/ginkgo/v2"
 )
@@ -32,12 +40,14 @@ type Container struct {
 	IsOptional       bool
 	RunDetached      bool
 	Name             string
+	ID               string
 	Image            string
 	ExtraRunningArgs string
 	Volumes          map[string]Volume
 	EnvVars          map[string]string
 	VppInstance      *VppInstance
 	AllocatedCpus    []int
+	ctx              context.Context
 }
 
 func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error) {
@@ -52,6 +62,7 @@ func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error
 	container.EnvVars = make(map[string]string)
 	container.Name = containerName
 	container.Suite = suite
+	container.ctx = context.Background()
 
 	if Image, ok := yamlInput["image"]; ok {
 		container.Image = Image.(string)
@@ -133,8 +144,6 @@ func (c *Container) GetContainerWorkDir() (res string) {
 
 func (c *Container) getContainerArguments() string {
 	args := "--ulimit nofile=90000:90000 --cap-add=all --privileged --network host"
-	c.allocateCpus()
-	args += fmt.Sprintf(" --cpuset-cpus=\"%d-%d\"", c.AllocatedCpus[0], c.AllocatedCpus[len(c.AllocatedCpus)-1])
 	args += c.getVolumesAsCliOption()
 	args += c.getEnvVarsAsCliOption()
 	if *VppSourceFileDir != "" {
@@ -145,22 +154,57 @@ func (c *Container) getContainerArguments() string {
 	return args
 }
 
-func (c *Container) runWithRetry(cmd string) error {
-	nTries := 5
-	for i := 0; i < nTries; i++ {
-		err := exechelper.Run(cmd)
-		if err == nil {
-			return nil
-		}
-		time.Sleep(1 * time.Second)
-	}
-	return fmt.Errorf("failed to run container command")
+func (c *Container) PullDockerImage(name string, ctx context.Context) {
+	// "func (*Client) ImagePull" doesn't work, returns "No such image"
+	c.Suite.Log("Pulling image: " + name)
+	_, err := exechelper.CombinedOutput("docker pull " + name)
+	c.Suite.AssertNil(err)
 }
 
+// Creates a container
 func (c *Container) Create() error {
-	cmd := "docker create " + c.getContainerArguments()
-	c.Suite.Log(cmd)
-	return exechelper.Run(cmd)
+	var sliceOfImageNames []string
+	images, err := c.Suite.Docker.ImageList(c.ctx, image.ListOptions{})
+	c.Suite.AssertNil(err)
+
+	for _, image := range images {
+		sliceOfImageNames = append(sliceOfImageNames, strings.Split(image.RepoTags[0], ":")[0])
+	}
+	if !slices.Contains(sliceOfImageNames, c.Image) {
+		c.PullDockerImage(c.Image, c.ctx)
+	}
+
+	c.allocateCpus()
+	cpuSet := fmt.Sprintf("%d-%d", c.AllocatedCpus[0], c.AllocatedCpus[len(c.AllocatedCpus)-1])
+	resp, err := c.Suite.Docker.ContainerCreate(
+		c.ctx,
+		&containerTypes.Config{
+			Image: c.Image,
+			Env:   c.getEnvVars(),
+			Cmd:   strings.Split(c.ExtraRunningArgs, " "),
+		},
+		&containerTypes.HostConfig{
+			Resources: containerTypes.Resources{
+				Ulimits: []*units.Ulimit{
+					{
+						Name: "nofile",
+						Soft: 90000,
+						Hard: 90000,
+					},
+				},
+				CpusetCpus: cpuSet,
+			},
+			CapAdd:      []string{"ALL"},
+			Privileged:  true,
+			NetworkMode: "host",
+			Binds:       c.getVolumesAsSlice(),
+		},
+		nil,
+		nil,
+		c.Name,
+	)
+	c.ID = resp.ID
+	return err
 }
 
 func (c *Container) allocateCpus() {
@@ -169,10 +213,45 @@ func (c *Container) allocateCpus() {
 	c.Suite.Log("Allocated CPUs " + fmt.Sprint(c.AllocatedCpus) + " to container " + c.Name)
 }
 
+// Starts a container
 func (c *Container) Start() error {
-	cmd := "docker start " + c.Name
-	c.Suite.Log(cmd)
-	return c.runWithRetry(cmd)
+	var err error
+	for nTries := 0; nTries < 5; nTries++ {
+		err = c.Suite.Docker.ContainerStart(c.ctx, c.ID, containerTypes.StartOptions{})
+		if err == nil {
+			continue
+		}
+		c.Suite.Log("Error while starting " + c.Name + ". Retrying...")
+		time.Sleep(1 * time.Second)
+	}
+
+	return err
+}
+
+func (c *Container) GetOutput() (string, string) {
+	// Wait for the container to finish executing
+	statusCh, errCh := c.Suite.Docker.ContainerWait(c.ctx, c.ID, containerTypes.WaitConditionNotRunning)
+	select {
+	case err := <-errCh:
+		c.Suite.AssertNil(err)
+	case <-statusCh:
+	}
+
+	// Get the logs from the container
+	logOptions := containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true}
+	logReader, err := c.Suite.Docker.ContainerLogs(c.ctx, c.ID, logOptions)
+	c.Suite.AssertNil(err)
+	defer logReader.Close()
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+
+	// Use stdcopy.StdCopy to demultiplex the multiplexed stream
+	_, err = stdcopy.StdCopy(&stdoutBuf, &stderrBuf, logReader)
+	c.Suite.AssertNil(err)
+
+	stdout := stdoutBuf.String()
+	stderr := stderrBuf.String()
+	return stdout, stderr
 }
 
 func (c *Container) prepareCommand() (string, error) {
@@ -180,7 +259,7 @@ func (c *Container) prepareCommand() (string, error) {
 		return "", fmt.Errorf("run container failed: name is blank")
 	}
 
-	cmd := "docker run "
+	cmd := "docker exec "
 	if c.RunDetached {
 		cmd += " -d"
 	}
@@ -201,12 +280,10 @@ func (c *Container) CombinedOutput() (string, error) {
 	return string(byteOutput), err
 }
 
-func (c *Container) Run() error {
-	cmd, err := c.prepareCommand()
-	if err != nil {
-		return err
-	}
-	return c.runWithRetry(cmd)
+// Creates and starts a container
+func (c *Container) Run() {
+	c.Suite.AssertNil(c.Create())
+	c.Suite.AssertNil(c.Start())
 }
 
 func (c *Container) addVolume(hostDir string, containerDir string, isDefaultWorkDir bool) {
@@ -217,6 +294,22 @@ func (c *Container) addVolume(hostDir string, containerDir string, isDefaultWork
 	c.Volumes[hostDir] = volume
 }
 
+func (c *Container) getVolumesAsSlice() []string {
+	var volumeSlice []string
+
+	if *VppSourceFileDir != "" {
+		volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", *VppSourceFileDir, *VppSourceFileDir))
+	}
+
+	if len(c.Volumes) > 0 {
+		for _, volume := range c.Volumes {
+			volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", volume.HostDir, volume.ContainerDir))
+		}
+	}
+
+	return volumeSlice
+}
+
 func (c *Container) getVolumesAsCliOption() string {
 	cliOption := ""
 
@@ -245,6 +338,18 @@ func (c *Container) getEnvVarsAsCliOption() string {
 	return cliOption
 }
 
+func (c *Container) getEnvVars() []string {
+	var envVars []string
+	if len(c.EnvVars) == 0 {
+		return envVars
+	}
+
+	for name, value := range c.EnvVars {
+		envVars = append(envVars, fmt.Sprintf("%s=%s", name, value))
+	}
+	return envVars
+}
+
 func (c *Container) newVppInstance(cpus []int, additionalConfigs ...Stanza) (*VppInstance, error) {
 	vpp := new(VppInstance)
 	vpp.Container = c
@@ -317,33 +422,48 @@ func (c *Container) getLogDirPath() string {
 func (c *Container) saveLogs() {
 	testLogFilePath := c.getLogDirPath() + "container-" + c.Name + ".log"
 
-	cmd := exec.Command("docker", "logs", "--details", "-t", c.Name)
-	c.Suite.Log(cmd)
-	output, err := cmd.CombinedOutput()
+	logs, err := c.log(0)
 	if err != nil {
 		c.Suite.Log(err)
+		return
 	}
 
 	f, err := os.Create(testLogFilePath)
 	if err != nil {
-		Fail("file create error: " + fmt.Sprint(err))
+		c.Suite.Log(err)
+		return
 	}
-	fmt.Fprint(f, string(output))
-	f.Close()
+	defer f.Close()
+	fmt.Fprint(f, logs)
 }
 
-// Outputs logs from docker containers. Set 'maxLines' to 0 to output the full log.
+// Returns logs from docker containers. Set 'maxLines' to 0 to output the full log.
 func (c *Container) log(maxLines int) (string, error) {
-	var cmd string
+	var logOptions containerTypes.LogsOptions
 	if maxLines == 0 {
-		cmd = "docker logs " + c.Name
+		logOptions = containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true, Details: true}
 	} else {
-		cmd = fmt.Sprintf("docker logs --tail %d %s", maxLines, c.Name)
+		logOptions = containerTypes.LogsOptions{ShowStdout: true, ShowStderr: true, Details: true, Tail: strconv.Itoa(maxLines)}
 	}
 
-	c.Suite.Log(cmd)
-	o, err := exechelper.CombinedOutput(cmd)
-	return string(o), err
+	out, err := c.Suite.Docker.ContainerLogs(c.ctx, c.ID, logOptions)
+	if err != nil {
+		c.Suite.Log(err)
+		return "", err
+	}
+	defer out.Close()
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+
+	_, err = stdcopy.StdCopy(&stdoutBuf, &stderrBuf, out)
+	if err != nil {
+		c.Suite.Log(err)
+	}
+
+	stdout := stdoutBuf.String()
+	stderr := stderrBuf.String()
+
+	return stdout + " " + stderr, err
 }
 
 func (c *Container) stop() error {
@@ -353,8 +473,13 @@ func (c *Container) stop() error {
 	}
 	c.VppInstance = nil
 	c.saveLogs()
-	c.Suite.Log("docker stop " + c.Name + " -t 0")
-	return exechelper.Run("docker stop " + c.Name + " -t 0")
+
+	c.Suite.Log("Stopping container " + c.Name)
+	timeout := 0
+	if err := c.Suite.Docker.ContainerStop(c.ctx, c.ID, containerTypes.StopOptions{Timeout: &timeout}); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (c *Container) CreateConfig(targetConfigName string, templateName string, values any) {
diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go
index ac255b9fd44..975e01d5b8e 100644
--- a/extras/hs-test/infra/hst_suite.go
+++ b/extras/hs-test/infra/hst_suite.go
@@ -14,10 +14,11 @@ import (
 	"strings"
 	"time"
 
+	containerTypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
 	"github.com/onsi/gomega/gmeasure"
 	"gopkg.in/yaml.v3"
 
-	"github.com/edwarnicke/exechelper"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -52,6 +53,7 @@ type HstSuite struct {
 	ProcessIndex      string
 	Logger            *log.Logger
 	LogFile           *os.File
+	Docker            *client.Client
 }
 
 func getTestFilename() string {
@@ -59,8 +61,16 @@ func getTestFilename() string {
 	return filepath.Base(filename)
 }
 
+func (s *HstSuite) newDockerClient() {
+	var err error
+	s.Docker, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	s.AssertNil(err)
+	s.Log("docker client created")
+}
+
 func (s *HstSuite) SetupSuite() {
 	s.CreateLogger()
+	s.newDockerClient()
 	s.Log("Suite Setup")
 	RegisterFailHandler(func(message string, callerSkip ...int) {
 		s.HstFail()
@@ -94,6 +104,7 @@ func (s *HstSuite) AddCpuContext(cpuCtx *CpuContext) {
 
 func (s *HstSuite) TearDownSuite() {
 	defer s.LogFile.Close()
+	defer s.Docker.Close()
 	s.Log("Suite Teardown")
 	s.UnconfigureNetworkTopology()
 }
@@ -104,7 +115,6 @@ func (s *HstSuite) TearDownTest() {
 		return
 	}
 	s.ResetContainers()
-	s.RemoveVolumes()
 
 	if s.Ip4AddrAllocator != nil {
 		s.Ip4AddrAllocator.DeleteIpAddresses()
@@ -121,18 +131,9 @@ func (s *HstSuite) SetupTest() {
 	s.Log("Test Setup")
 	s.StartedContainers = s.StartedContainers[:0]
 	s.SkipIfUnconfiguring()
-	s.SetupVolumes()
 	s.SetupContainers()
 }
 
-func (s *HstSuite) SetupVolumes() {
-	for _, volume := range s.Volumes {
-		cmd := "docker volume create --name=" + volume
-		s.Log(cmd)
-		exechelper.Run(cmd)
-	}
-}
-
 func (s *HstSuite) SetupContainers() {
 	for _, container := range s.Containers {
 		if !container.IsOptional {
@@ -214,6 +215,10 @@ func (s *HstSuite) AssertNotContains(testString, contains interface{}, msgAndArg
 	Expect(testString).ToNot(ContainSubstring(fmt.Sprint(contains)), msgAndArgs...)
 }
 
+func (s *HstSuite) AssertEmpty(object interface{}, msgAndArgs ...interface{}) {
+	Expect(object).To(BeEmpty(), msgAndArgs...)
+}
+
 func (s *HstSuite) AssertNotEmpty(object interface{}, msgAndArgs ...interface{}) {
 	Expect(object).ToNot(BeEmpty(), msgAndArgs...)
 }
@@ -283,15 +288,10 @@ func (s *HstSuite) SkipUnlessExtendedTestsBuilt() {
 func (s *HstSuite) ResetContainers() {
 	for _, container := range s.StartedContainers {
 		container.stop()
-		exechelper.Run("docker rm " + container.Name)
-	}
-}
-
-func (s *HstSuite) RemoveVolumes() {
-	for _, volumeName := range s.Volumes {
-		cmd := "docker volume rm " + volumeName
-		exechelper.Run(cmd)
-		os.RemoveAll(volumeName)
+		s.Log("Removing container " + container.Name)
+		if err := s.Docker.ContainerRemove(container.ctx, container.ID, containerTypes.RemoveOptions{RemoveVolumes: true}); err != nil {
+			s.Log(err)
+		}
 	}
 }
 
diff --git a/extras/hs-test/infra/suite_nginx.go b/extras/hs-test/infra/suite_nginx.go
index bb1bdb0f42b..3a8b28e52eb 100644
--- a/extras/hs-test/infra/suite_nginx.go
+++ b/extras/hs-test/infra/suite_nginx.go
@@ -66,7 +66,7 @@ func (s *NginxSuite) SetupTest() {
 	s.AssertNil(proxyVpp.createTap(serverInterface, 2))
 
 	nginxContainer := s.GetTransientContainerByName(NginxProxyContainerName)
-	nginxContainer.Create()
+	s.AssertNil(nginxContainer.Create())
 
 	values := struct {
 		Proxy  string