hs-test: test vpp+nginx mirroring with tap ifaces

Type: test
Signed-off-by: Maros Ondrejicka <mondreji@cisco.com>
Change-Id: I05bbed8fd9d40929f040574044aed5292a475e91

13 files changed, 281 insertions, 16 deletions

diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go
index c55fddead54..5def2789528 100644
--- a/extras/hs-test/container.go
+++ b/extras/hs-test/container.go
@@ -121,12 +121,22 @@ func (c *Container) GetContainerWorkDir() (res string) {
 	return
 }
 
-func (c *Container) getRunCommand() string {
-	cmd := "docker run --cap-add=all -d --privileged --network host --rm"
-	cmd += c.getVolumesAsCliOption()
-	cmd += c.getEnvVarsAsCliOption()
-	cmd += " --name " + c.name + " " + c.image + " " + c.extraRunningArgs
-	return cmd
+func (c *Container) getContainerArguments() string {
+	args := "--cap-add=all --privileged --network host --rm"
+	args += c.getVolumesAsCliOption()
+	args += c.getEnvVarsAsCliOption()
+	args += " --name " + c.name + " " + c.image
+	return args
+}
+
+func (c *Container) create() {
+	cmd := "docker create " + c.getContainerArguments()
+	exechelper.Run(cmd)
+}
+
+func (c *Container) start() {
+	cmd := "docker start " + c.name
+	exechelper.Run(cmd)
 }
 
 func (c *Container) run() error {
@@ -134,7 +144,8 @@ func (c *Container) run() error {
 		return fmt.Errorf("run container failed: name is blank")
 	}
 
-	cmd := c.getRunCommand()
+	cmd := "docker run -d " + c.getContainerArguments() + " " + c.extraRunningArgs
+	c.Suite().log(cmd)
 	err := exechelper.Run(cmd)
 	if err != nil {
 		return fmt.Errorf("container run failed: %s", err)
@@ -273,6 +284,14 @@ func (c *Container) saveLogs() {
 	f.Close()
 }
 
+func (c *Container) log() string {
+	cmd := "docker logs " + c.name
+	c.Suite().log(cmd)
+	o, err := exechelper.CombinedOutput(cmd)
+	c.Suite().assertNil(err)
+	return string(o)
+}
+
 func (c *Container) stop() error {
 	if c.vppInstance != nil && c.vppInstance.apiChannel != nil {
 		c.vppInstance.saveLogs()
diff --git a/extras/hs-test/docker/Dockerfile.nginx-server b/extras/hs-test/docker/Dockerfile.nginx-server
new file mode 100644
index 00000000000..1971158131b
--- /dev/null
+++ b/extras/hs-test/docker/Dockerfile.nginx-server
@@ -0,0 +1,12 @@
+ARG UBUNTU_VERSION
+
+FROM ubuntu:${UBUNTU_VERSION}
+
+RUN apt-get update \
+ && apt-get install -y nginx \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY resources/nginx/nginx_server_mirroring.conf /nginx.conf
+
+
+ENTRYPOINT ["nginx", "-c", "/nginx.conf"]
diff --git a/extras/hs-test/framework_test.go b/extras/hs-test/framework_test.go
index fdce5c45bf7..84aa570e681 100644
--- a/extras/hs-test/framework_test.go
+++ b/extras/hs-test/framework_test.go
@@ -25,3 +25,8 @@ func TestNoTopo(t *testing.T) {
 	var m NoTopoSuite
 	suite.Run(t, &m)
 }
+
+func TestNginx(t *testing.T) {
+	var m NginxSuite
+	suite.Run(t, &m)
+}
diff --git a/extras/hs-test/http_test.go b/extras/hs-test/http_test.go
index 96985be1ca7..c1b3b7f2a03 100644
--- a/extras/hs-test/http_test.go
+++ b/extras/hs-test/http_test.go
@@ -48,8 +48,7 @@ func (s *NoTopoSuite) TestNginxAsServer() {
 	s.assertNil(nginxCont.run())
 
 	vpp := s.getContainerByName("vpp").vppInstance
-	err := vpp.waitForApp("-app", 5)
-	s.assertNil(err)
+	vpp.waitForApp("-app", 5)
 
 	serverAddress := s.netInterfaces[tapInterfaceName].Peer().IP4AddressString()
 
@@ -86,8 +85,7 @@ func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string) error {
 
 	nginxCont := s.getContainerByName("nginx")
 	s.assertNil(nginxCont.run())
-	err := vpp.waitForApp("-app", 5)
-	s.assertNil(err)
+	vpp.waitForApp("-app", 5)
 
 	cmd := exec.Command(exeName, args...)
 	s.log(cmd)
diff --git a/extras/hs-test/mirroring_test.go b/extras/hs-test/mirroring_test.go
new file mode 100644
index 00000000000..97c6c8dbf45
--- /dev/null
+++ b/extras/hs-test/mirroring_test.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"github.com/edwarnicke/exechelper"
+)
+
+func (s *NginxSuite) TestMirroring() {
+	proxyAddress := s.netInterfaces[mirroringClientInterfaceName].Peer().IP4AddressString()
+
+	path := "/64B.json"
+
+	testCommand := "wrk -c 20 -t 10 -d 40 http://" + proxyAddress + ":80" + path
+	s.log(testCommand)
+	o, _ := exechelper.Output(testCommand)
+	s.log(string(o))
+	s.assertNotEmpty(o)
+
+	// Check if log output from VPP contains 'no lcl port' warnings
+	// TODO: Need to change after adding session worker counter
+	vppProxyContainer := s.getContainerByName(vppProxyContainerName)
+	logContent := vppProxyContainer.log()
+	s.assertNotContains(logContent, "no lcl port")
+}
diff --git a/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf b/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf
new file mode 100644
index 00000000000..03af8b76f15
--- /dev/null
+++ b/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf
@@ -0,0 +1,82 @@
+user root;
+worker_processes 4;
+worker_rlimit_nofile 102400;
+daemon off;
+
+events {
+  use epoll;
+  worker_connections 102400;
+  accept_mutex off;
+}
+
+http {
+  include mime.types;
+  default_type application/octet-stream;
+
+  access_log off;
+
+  keepalive_timeout 300;
+  keepalive_requests 1000000;
+
+  proxy_connect_timeout 300;
+  large_client_header_buffers 4 512k;
+  client_max_body_size 3000m;
+  client_header_buffer_size 2048m;
+  client_body_buffer_size 1024m;
+  proxy_buffers 16 10240k;
+  proxy_buffer_size 10240k;
+
+  gzip on;
+
+  upstream bk {
+    server 10.10.2.1:8091;
+    keepalive 30000;
+  }
+  upstream bk1 {
+    server 10.10.2.1:8092;
+    keepalive 30000;
+  }
+  upstream bk2 {
+    server 10.10.2.1:8093;
+    keepalive 30000;
+  }
+
+  server {
+    listen 80;
+    server_name 10.10.1.2;
+
+    server_tokens off;
+
+    proxy_redirect off;
+
+    location / {
+      root html;
+      index index.html index.htm;
+      proxy_pass http://bk;
+      proxy_set_header Connection "";
+      proxy_set_header X-Original-URI $request_uri;
+      proxy_set_header Host $host:$server_port;
+      chunked_transfer_encoding on;
+      proxy_http_version 1.1;
+      mirror /mimic1;
+      mirror /mimic2;
+      mirror_request_body on;
+    }
+    location /mimic1 {
+      proxy_pass http://bk1$request_uri;
+      proxy_set_header X-Original-URI $request_uri;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding on;
+      proxy_http_version 1.1;
+      proxy_set_header Host $host:$server_port;
+    }
+    location /mimic2 {
+      proxy_pass http://bk2$request_uri;
+      proxy_set_header X-Original-URI $request_uri;
+      proxy_set_header Host $host:$server_port;
+      proxy_set_header Connection "";
+      proxy_http_version 1.1;
+      chunked_transfer_encoding on;
+    }
+  }
+}
diff --git a/extras/hs-test/resources/nginx/nginx_server_mirroring.conf b/extras/hs-test/resources/nginx/nginx_server_mirroring.conf
new file mode 100644
index 00000000000..4056801ea13
--- /dev/null
+++ b/extras/hs-test/resources/nginx/nginx_server_mirroring.conf
@@ -0,0 +1,32 @@
+master_process on;
+worker_rlimit_nofile 10240;
+worker_processes 2;
+daemon off;
+
+events {
+  use epoll;
+  worker_connections  10240;
+  accept_mutex       off;
+  multi_accept       off;
+}
+
+http {
+  keepalive_timeout 300s;
+  keepalive_requests 1000000;
+  sendfile on;
+  server {
+    listen 8091;
+    listen 8092;
+    listen 8093;
+    root /usr/share/nginx;
+    index index.html index.htm;
+    location /return_ok
+    {
+      return 200 '';
+    }
+    location /64B.json
+    {
+      return 200 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx';
+    }
+  }
+}
diff --git a/extras/hs-test/script/build.sh b/extras/hs-test/script/build.sh
index 78facd1605d..97278eb4919 100755
--- a/extras/hs-test/script/build.sh
+++ b/extras/hs-test/script/build.sh
@@ -61,3 +61,4 @@ docker_build () {
 
 docker_build hs-test/vpp vpp
 docker_build hs-test/nginx-ldp nginx
+docker_build hs-test/nginx-server nginx-server
diff --git a/extras/hs-test/suite_nginx_test.go b/extras/hs-test/suite_nginx_test.go
new file mode 100644
index 00000000000..cd67efb6cc5
--- /dev/null
+++ b/extras/hs-test/suite_nginx_test.go
@@ -0,0 +1,50 @@
+package main
+
+const (
+	// These correspond to names used in yaml config
+	mirroringClientInterfaceName = "hst_client"
+	mirroringServerInterfaceName = "hst_server"
+	vppProxyContainerName        = "vpp-proxy"
+	nginxProxyContainerName      = "nginx-proxy"
+	nginxServerContainerName     = "nginx-server"
+)
+
+type NginxSuite struct {
+	HstSuite
+}
+
+func (s *NginxSuite) SetupSuite() {
+	s.loadNetworkTopology("2taps")
+
+	s.loadContainerTopology("nginxProxyAndServer")
+}
+
+func (s *NginxSuite) SetupTest() {
+	s.SetupVolumes()
+	s.SetupContainers()
+
+	// Setup test conditions
+	var startupConfig Stanza
+	startupConfig.
+		NewStanza("session").
+		Append("enable").
+		Append("use-app-socket-api").Close()
+
+	// ... for proxy
+	vppProxyContainer := s.getContainerByName(vppProxyContainerName)
+	proxyVpp, _ := vppProxyContainer.newVppInstance(startupConfig)
+	proxyVpp.start()
+
+	clientInterface := s.netInterfaces[mirroringClientInterfaceName]
+	proxyVpp.createTap(clientInterface, 1)
+
+	serverInterface := s.netInterfaces[mirroringServerInterfaceName]
+	proxyVpp.createTap(serverInterface, 2)
+
+	nginxContainer := s.getTransientContainerByName(nginxProxyContainerName)
+	nginxContainer.create()
+	nginxContainer.copy("./resources/nginx/nginx_proxy_mirroring.conf", "/nginx.conf")
+	nginxContainer.start()
+
+	proxyVpp.waitForApp("-app", 5)
+}
diff --git a/extras/hs-test/suite_no_topo_test.go b/extras/hs-test/suite_no_topo_test.go
index bca1dbf1828..9051284c780 100644
--- a/extras/hs-test/suite_no_topo_test.go
+++ b/extras/hs-test/suite_no_topo_test.go
@@ -35,5 +35,5 @@ func (s *NoTopoSuite) SetupTest() {
 
 	tapInterface := s.netInterfaces[tapInterfaceName]
 
-	vpp.createTap(1, tapInterface)
+	vpp.createTap(tapInterface)
 }
diff --git a/extras/hs-test/topo-containers/nginxProxyAndServer.yaml b/extras/hs-test/topo-containers/nginxProxyAndServer.yaml
new file mode 100644
index 00000000000..bac6a2df174
--- /dev/null
+++ b/extras/hs-test/topo-containers/nginxProxyAndServer.yaml
@@ -0,0 +1,20 @@
+---
+volumes:
+  - volume: &shared-vol-proxy
+      host-dir: /tmp/shared-vol-proxy
+
+containers:
+  - name: "vpp-proxy"
+    volumes:
+      - <<: *shared-vol-proxy
+        container-dir: "/tmp/vpp"
+        is-default-work-dir: true
+  - name: "nginx-proxy"
+    volumes:
+      - <<: *shared-vol-proxy
+        container-dir: "/tmp/nginx"
+        is-default-work-dir: true
+    image: "hs-test/nginx-ldp"
+    is-optional: true
+  - name: "nginx-server"
+    image: "hs-test/nginx-server"
diff --git a/extras/hs-test/topo-network/2taps.yaml b/extras/hs-test/topo-network/2taps.yaml
new file mode 100644
index 00000000000..38f6fca4f31
--- /dev/null
+++ b/extras/hs-test/topo-network/2taps.yaml
@@ -0,0 +1,18 @@
+---
+devices:
+  - name: "hst_client"
+    type: "tap"
+    ip4:
+      network: 1
+    peer:
+      name: ""
+      ip4:
+        network: 1
+  - name: "hst_server"
+    type: "tap"
+    ip4:
+      network: 2
+    peer:
+      name: ""
+      ip4:
+        network: 2
diff --git a/extras/hs-test/vppinstance.go b/extras/hs-test/vppinstance.go
index 29b86d5adcb..4092d35cfd6 100644
--- a/extras/hs-test/vppinstance.go
+++ b/extras/hs-test/vppinstance.go
@@ -160,15 +160,16 @@ func (vpp *VppInstance) vppctl(command string, arguments ...any) string {
 	return string(output)
 }
 
-func (vpp *VppInstance) waitForApp(appName string, timeout int) error {
+func (vpp *VppInstance) waitForApp(appName string, timeout int) {
 	for i := 0; i < timeout; i++ {
 		o := vpp.vppctl("show app")
 		if strings.Contains(o, appName) {
-			return nil
+			return
 		}
 		time.Sleep(1 * time.Second)
 	}
-	return fmt.Errorf("timeout while waiting for app '%s'", appName)
+	vpp.Suite().assertNil(1, "Timeout while waiting for app '%s'", appName)
+	return
 }
 
 func (vpp *VppInstance) createAfPacket(
@@ -253,9 +254,13 @@ func (vpp *VppInstance) addAppNamespace(
 }
 
 func (vpp *VppInstance) createTap(
-	id uint32,
 	tap *NetInterface,
+	tapId ...uint32,
 ) error {
+	var id uint32 = 1
+	if len(tapId) > 0 {
+		id = tapId[0]
+	}
 	createTapReq := &tapv2.TapCreateV2{
 		ID:               id,
 		HostIfNameSet:    true,