hs-test: add test suite features

Test suite now supports assertions which on fail stop test case run,
also it allows to create docker containers which are going to be
stopped automatically after the test run is finished.

Type: improvement
Signed-off-by: Maros Ondrejicka <maros.ondrejicka@pantheon.tech>
Change-Id: I2834709b1efd17b8182d36cc0404b986b4ed595d
Signed-off-by: Filip Tehlar <ftehlar@cisco.com>

10 files changed, 261 insertions, 77 deletions

diff --git a/extras/hs-test/README.rst b/extras/hs-test/README.rst
index dfaae44eeff..7a99621c8dc 100755
--- a/extras/hs-test/README.rst
+++ b/extras/hs-test/README.rst
@@ -45,10 +45,11 @@ For adding a new suite, please see `Modifying the framework`_ below.
 #. Implement test behaviour inside the test method. This typically includes the following:
 
   #. Start docker container(s) as needed. Function ``dockerRun(instance, args string)``
-     from ``utils.go`` serves this purpose. Alternatively use suite struct's ``NewContainer(name string)`` method
+     from ``utils.go`` serves this purpose. Alternatively use suite struct's ``NewContainer(name string)`` method to create
+     an object representing a container and start it with ``run()`` method
   #. Execute *hs-test* action(s) inside any of the running containers.
      Function ``hstExec`` from ``utils.go`` does this by using ``docker exec`` command to run ``hs-test`` executable.
-     For starting an VPP instance inside a container, the ``Vpp`` struct can be used as a forward-looking alternative
+     For starting an VPP instance inside a container, the ``VppInstance`` struct can be used as a forward-looking alternative
   #. Run arbitrary commands inside the containers with ``dockerExec(cmd string, instance string)``
   #. Run other external tool with one of the preexisting functions in the ``utils.go`` file.
      For example, use ``wget`` with ``startWget(..)`` function
@@ -124,14 +125,13 @@ Modifying the framework
 
 #. Adding a new suite takes place in ``framework_test.go``
 
-#. Make a ``struct`` with at least ``HstSuite`` struct and a ``teardownSuite`` function as its members.
+#. Make a ``struct`` with at least ``HstSuite`` struct as its member.
    HstSuite provides functionality that can be shared for all suites, like starting containers
 
         ::
 
                 type MySuite struct {
                         HstSuite
-                        teardownSuite func()
                 }
 
 #. Implement SetupSuite method which testify runs before running the tests.
@@ -147,17 +147,6 @@ Modifying the framework
                         s.teardownSuite = setupSuite(&s.Suite, "myTopology")
                 }
 
-#. Implement TearDownSuite method which testify runs after the tests, to clean-up.
-   It's good idea to add at least the suite's own ``teardownSuite()``
-   and HstSuite upper suite's ``stopContainers()`` methods
-
-        ::
-
-                func (s *MySuite) TearDownSuite() {
-                        s.teardownSuite()
-                        s.StopContainers()
-                }
-
 #. In order for ``go test`` to run this suite, we need to create a normal test function and pass our suite to ``suite.Run``
 
         ::
diff --git a/extras/hs-test/actions.go b/extras/hs-test/actions.go
index 9885f87b87d..fe07b5fdb39 100755
--- a/extras/hs-test/actions.go
+++ b/extras/hs-test/actions.go
@@ -236,20 +236,26 @@ func (a *Actions) Configure2Veths(args []string) *ActionResult {
 
 	ctx, cancel := newVppContext()
 	defer cancel()
+
+	vppConfig, err := DeserializeVppConfig(args[2])
+	if err != nil {
+		return NewActionResult(err, ActionResultWithDesc("deserializing configuration failed"))
+	}
+
 	con, vppErrCh := vpphelper.StartAndDialContext(ctx,
-		vpphelper.WithVppConfig(configTemplate+startup.ToString()),
+		vpphelper.WithVppConfig(vppConfig.getTemplate()+startup.ToString()),
 		vpphelper.WithRootDir(fmt.Sprintf("/tmp/%s", args[1])))
 	exitOnErrCh(ctx, cancel, vppErrCh)
 
 	var fn func(context.Context, api.Connection) error
-	if args[2] == "srv" {
+	if vppConfig.Variant == "srv" {
 		fn = configure2vethsTopo("vppsrv", "10.10.10.1/24", "1", 1)
-	} else if args[2] == "srv-with-preset-hw-addr" {
+	} else if vppConfig.Variant == "srv-with-preset-hw-addr" {
 		fn = configure2vethsTopo("vppsrv", "10.10.10.1/24", "1", 1, "00:00:5e:00:53:01")
 	} else {
 		fn = configure2vethsTopo("vppcln", "10.10.10.2/24", "2", 2)
 	}
-	err := fn(ctx, con)
+	err = fn(ctx, con)
 	if err != nil {
 		return NewActionResult(err, ActionResultWithDesc("configuration failed"))
 	}
diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go
new file mode 100644
index 00000000000..3128a8ecdd5
--- /dev/null
+++ b/extras/hs-test/container.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/edwarnicke/exechelper"
+)
+
+type Container struct {
+	name string
+}
+
+func (c *Container) run() error {
+	if c.name == "" {
+		return fmt.Errorf("create volume failed: container name is blank")
+	}
+
+	exechelper.Run(fmt.Sprintf("mkdir -p /tmp/%s/sync", c.name))
+	syncPath := fmt.Sprintf("-v /tmp/%s/sync:/tmp/sync", c.name)
+	cmd := "docker run --cap-add=all -d --privileged --network host --rm "
+	cmd += syncPath
+	cmd += " --name " + c.name + " hs-test/vpp"
+	fmt.Println(cmd)
+	err := exechelper.Run(cmd)
+	if err != nil {
+		return fmt.Errorf("create volume failed: %s", err)
+	}
+
+	return nil
+}
+
diff --git a/extras/hs-test/echo_test.go b/extras/hs-test/echo_test.go
index 74ff4cb0d6e..9f91e2afeb7 100755
--- a/extras/hs-test/echo_test.go
+++ b/extras/hs-test/echo_test.go
@@ -6,46 +6,26 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestEchoBuiltin() {
-	t := s.T()
+func (s *VethsSuite) TestEchoBuiltin() {
 	srvInstance := "echo-srv-internal"
 	clnInstance := "echo-cln-internal"
-	err := dockerRun(srvInstance, "")
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+
+	s.assertNil(dockerRun(srvInstance, ""), "failed to start docker (srv)")
 	defer func() { exechelper.Run("docker stop " + srvInstance) }()
 
-	err = dockerRun(clnInstance, "")
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(dockerRun(clnInstance, ""), "failed to start docker (cln)")
 	defer func() { exechelper.Run("docker stop " + clnInstance) }()
 
-	_, err = hstExec("Configure2Veths srv", srvInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	_, err := hstExec("Configure2Veths srv", srvInstance)
+	s.assertNil(err)
 
 	_, err = hstExec("Configure2Veths cln", clnInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 
 	_, err = hstExec("RunEchoSrvInternal private-segment-size 1g fifo-size 4 no-echo", srvInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 
 	o, err := hstExec("RunEchoClnInternal nclients 10000 bytes 1 syn-timeout 100 test-timeout 100 no-return private-segment-size 1g fifo-size 4", clnInstance)
-	if err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	s.assertNil(err)
 	fmt.Println(o)
 }
diff --git a/extras/hs-test/framework_test.go b/extras/hs-test/framework_test.go
index 38003925f2f..fc186f35180 100755
--- a/extras/hs-test/framework_test.go
+++ b/extras/hs-test/framework_test.go
@@ -1,53 +1,106 @@
 package main
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/edwarnicke/exechelper"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
 
-type TapSuite struct {
+type HstSuite struct {
 	suite.Suite
 	teardownSuite func()
+	containers    []string
+	volumes       []string
 }
 
-func (s *TapSuite) SetupSuite() {
-	time.Sleep(1 * time.Second)
-	s.teardownSuite = setupSuite(&s.Suite, "tap")
+func (s *HstSuite) TearDownSuite() {
+	s.teardownSuite()
+	s.StopContainers()
+	s.RemoveVolumes()
 }
 
-func (s *TapSuite) TearDownSuite() {
-	s.teardownSuite()
+func (s *HstSuite) hstFail() {
+	s.T().FailNow()
 }
 
-type Veths2Suite struct {
-	suite.Suite
-	teardownSuite func()
+func (s *HstSuite) assertNil(object interface{}, msgAndArgs ...interface{}) {
+	if !assert.Nil(s.T(), object, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertNotNil(object interface{}, msgAndArgs ...interface{}) {
+	if !assert.NotNil(s.T(), object, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertEqual(expected, actual interface{}, msgAndArgs ...interface{}) {
+	if !assert.Equal(s.T(), expected, actual, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) assertNotContains(testString, contains interface{}, msgAndArgs ...interface{}) {
+	if !assert.NotContains(s.T(), testString, contains, msgAndArgs...) {
+		s.hstFail()
+	}
+}
+
+func (s *HstSuite) NewContainer(name string) (*Container, error) {
+	if name == "" {
+		return nil, fmt.Errorf("creating container failed: name must not be blank")
+	}
+
+	s.containers = append(s.containers, name)
+
+	container := new(Container)
+	container.name = name
+	return container, nil
 }
 
-func (s *Veths2Suite) SetupSuite() {
+func (s *HstSuite) StopContainers() {
+	for _, containerName := range s.containers {
+		exechelper.Run("docker stop " + containerName)
+	}
+}
+
+func (s *HstSuite) RemoveVolumes() {
+	for _, volumeName := range s.volumes {
+		exechelper.Run("docker volume rm " + volumeName)
+	}
+}
+
+type TapSuite struct {
+	HstSuite
+}
+
+func (s *TapSuite) SetupSuite() {
 	time.Sleep(1 * time.Second)
-	s.teardownSuite = setupSuite(&s.Suite, "2peerVeth")
+	s.teardownSuite = setupSuite(&s.Suite, "tap")
 }
 
-func (s *Veths2Suite) TearDownSuite() {
-	s.teardownSuite()
+type VethsSuite struct {
+	HstSuite
+}
+
+func (s *VethsSuite) SetupSuite() {
+	time.Sleep(1 * time.Second)
+	s.teardownSuite = setupSuite(&s.Suite, "2peerVeth")
 }
 
 type NsSuite struct {
-	suite.Suite
-	teardownSuite func()
+	HstSuite
 }
 
 func (s *NsSuite) SetupSuite() {
 	s.teardownSuite = setupSuite(&s.Suite, "ns")
 }
 
-func (s *NsSuite) TearDownSuite() {
-	s.teardownSuite()
-}
-
 func setupSuite(s *suite.Suite, topologyName string) func() {
 	t := s.T()
 	topology, err := LoadTopology(TopologyDir, topologyName)
@@ -75,7 +128,7 @@ func TestNs(t *testing.T) {
 	suite.Run(t, &m)
 }
 
-func TestVeths2(t *testing.T) {
-	var m Veths2Suite
+func TestVeths(t *testing.T) {
+	var m VethsSuite
 	suite.Run(t, &m)
 }
diff --git a/extras/hs-test/http_test.go b/extras/hs-test/http_test.go
index bd93736376d..99b509fdcff 100755
--- a/extras/hs-test/http_test.go
+++ b/extras/hs-test/http_test.go
@@ -37,7 +37,7 @@ func (s *NsSuite) TestHttpTps() {
 	}
 }
 
-func (s *Veths2Suite) TestHttpCli() {
+func (s *VethsSuite) TestHttpCli() {
 	t := s.T()
 
 	srvInstance := "http-cli-srv"
diff --git a/extras/hs-test/ldp_test.go b/extras/hs-test/ldp_test.go
index 13c102e0633..c219c82ea50 100755
--- a/extras/hs-test/ldp_test.go
+++ b/extras/hs-test/ldp_test.go
@@ -8,7 +8,7 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestLDPreloadIperfVpp() {
+func (s *VethsSuite) TestLDPreloadIperfVpp() {
 	t := s.T()
 	var clnVclConf, srvVclConf Stanza
 
diff --git a/extras/hs-test/utils.go b/extras/hs-test/utils.go
index 3dc511ee13d..4dda4e462b0 100755
--- a/extras/hs-test/utils.go
+++ b/extras/hs-test/utils.go
@@ -16,6 +16,7 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
+// TODO remove `configTemplate` once its usage has been replaced everywhere with VppConfig
 const configTemplate = `unix {
   nodaemon
   log %[1]s/var/log/vpp/vpp.log
diff --git a/extras/hs-test/vcl_test.go b/extras/hs-test/vcl_test.go
index f699a65295c..e1d23bda020 100755
--- a/extras/hs-test/vcl_test.go
+++ b/extras/hs-test/vcl_test.go
@@ -7,21 +7,21 @@ import (
 	"github.com/edwarnicke/exechelper"
 )
 
-func (s *Veths2Suite) TestVclEchoQuic() {
+func (s *VethsSuite) TestVclEchoQuic() {
 	s.T().Skip("quic test skipping..")
 	s.testVclEcho("quic")
 }
 
-func (s *Veths2Suite) TestVclEchoUdp() {
+func (s *VethsSuite) TestVclEchoUdp() {
 	s.T().Skip("udp echo currently broken in vpp, skipping..")
 	s.testVclEcho("udp")
 }
 
-func (s *Veths2Suite) TestVclEchoTcp() {
+func (s *VethsSuite) TestVclEchoTcp() {
 	s.testVclEcho("tcp")
 }
 
-func (s *Veths2Suite) testVclEcho(proto string) {
+func (s *VethsSuite) testVclEcho(proto string) {
 	t := s.T()
 
 	exechelper.Run("docker volume create --name=echo-srv-vol")
@@ -86,12 +86,12 @@ func (s *Veths2Suite) testVclEcho(proto string) {
 	fmt.Println(o)
 }
 
-func (s *Veths2Suite) TestVclRetryAttach() {
+func (s *VethsSuite) TestVclRetryAttach() {
 	s.T().Skip()
 	s.testRetryAttach("tcp")
 }
 
-func (s *Veths2Suite) testRetryAttach(proto string) {
+func (s *VethsSuite) testRetryAttach(proto string) {
 	t := s.T()
 
 	exechelper.Run("docker volume create --name=echo-srv-vol")
diff --git a/extras/hs-test/vppinstance.go b/extras/hs-test/vppinstance.go
new file mode 100644
index 00000000000..c6d3935cc60
--- /dev/null
+++ b/extras/hs-test/vppinstance.go
@@ -0,0 +1,124 @@
+package main
+
+import (
+	"fmt"
+	"encoding/json"
+	"github.com/edwarnicke/exechelper"
+)
+
+const vppConfigTemplate = `unix {
+  nodaemon
+  log %[1]s/var/log/vpp/vpp.log
+  full-coredump
+  cli-listen %[1]s%[2]s
+  runtime-dir %[1]s/var/run
+  gid vpp
+}
+
+api-trace {
+  on
+}
+
+api-segment {
+  gid vpp
+}
+
+socksvr {
+  socket-name %[1]s/var/run/vpp/api.sock
+}
+
+statseg {
+  socket-name %[1]s/var/run/vpp/stats.sock
+}
+
+plugins {
+  plugin unittest_plugin.so { enable }
+  plugin dpdk_plugin.so { disable }
+  plugin crypto_aesni_plugin.so { enable }
+  plugin quic_plugin.so { enable }
+}
+
+`
+
+type VppInstance struct {
+	container *Container
+	config VppConfig
+	actionFuncName string
+}
+
+type VppConfig struct {
+	Variant string
+	CliSocketFilePath string
+}
+
+func (vc *VppConfig) getTemplate() string {
+	return fmt.Sprintf(vppConfigTemplate, "%[1]s", vc.CliSocketFilePath)
+}
+
+func (vpp *VppInstance) set2VethsServer() {
+	vpp.actionFuncName = "Configure2Veths"
+	vpp.config.Variant = "srv"
+}
+
+func (vpp *VppInstance) set2VethsClient() {
+	vpp.actionFuncName = "Configure2Veths"
+	vpp.config.Variant = "cln"
+}
+
+func (vpp *VppInstance) setCliSocket(filePath string) {
+	vpp.config.CliSocketFilePath = filePath
+}
+
+func (vpp *VppInstance) getCliSocket() string {
+	return fmt.Sprintf("/tmp/%s/%s", vpp.actionFuncName, vpp.config.CliSocketFilePath)
+}
+
+func (vpp *VppInstance) start() error {
+	if vpp.config.Variant == "" {
+		return fmt.Errorf("vpp start failed: variant must not be blank")
+	}
+	if vpp.actionFuncName == "" {
+		return fmt.Errorf("vpp start failed: action function name must not be blank")
+	}
+
+	serializedConfig, err := json.Marshal(vpp.config)
+	if err != nil {
+		return fmt.Errorf("vpp start failed: serializing configuration failed: %s", err)
+	}
+	args := fmt.Sprintf("%s '%s'", vpp.actionFuncName, string(serializedConfig))
+	_, err = hstExec(args, vpp.container.name)
+	if err != nil {
+		return fmt.Errorf("vpp start failed: %s", err)
+	}
+
+	return nil
+}
+
+func (vpp *VppInstance) vppctl(command string) (string, error) {
+	dockerExecCommand := fmt.Sprintf("docker exec --detach=false %[1]s vppctl -s %[2]s %[3]s",
+		vpp.container.name, vpp.getCliSocket(), command)
+	output, err := exechelper.CombinedOutput(dockerExecCommand)
+	if err != nil {
+		return "", fmt.Errorf("vppctl failed: %s", err)
+	}
+
+	return string(output), nil
+}
+
+func NewVppInstance(c *Container) *VppInstance {
+	vpp := new(VppInstance)
+	vpp.container = c
+	return vpp
+}
+
+func DeserializeVppConfig(input string) (VppConfig, error) {
+	var vppConfig VppConfig
+	err := json.Unmarshal([]byte(input), &vppConfig)
+	if err != nil {
+		// Since input is not a  valid JSON it is going be used as variant value
+		// for compatibility reasons
+		vppConfig.Variant = input
+		vppConfig.CliSocketFilePath = "/var/run/vpp/cli.sock"
+	}
+	return vppConfig, nil
+}