From c2f76f4590f57729d1bcf03bd816c10991431b18 Mon Sep 17 00:00:00 2001 From: Maros Ondrejicka Date: Mon, 27 Feb 2023 13:22:45 +0100 Subject: hs-test: test vpp+nginx mirroring with tap ifaces Type: test Signed-off-by: Maros Ondrejicka Change-Id: I05bbed8fd9d40929f040574044aed5292a475e91 --- extras/hs-test/container.go | 33 +++++++-- extras/hs-test/docker/Dockerfile.nginx-server | 12 ++++ extras/hs-test/framework_test.go | 5 ++ extras/hs-test/http_test.go | 6 +- extras/hs-test/mirroring_test.go | 23 ++++++ .../resources/nginx/nginx_proxy_mirroring.conf | 82 ++++++++++++++++++++++ .../resources/nginx/nginx_server_mirroring.conf | 32 +++++++++ extras/hs-test/script/build.sh | 1 + extras/hs-test/suite_nginx_test.go | 50 +++++++++++++ extras/hs-test/suite_no_topo_test.go | 2 +- .../topo-containers/nginxProxyAndServer.yaml | 20 ++++++ extras/hs-test/topo-network/2taps.yaml | 18 +++++ extras/hs-test/vppinstance.go | 13 ++-- 13 files changed, 281 insertions(+), 16 deletions(-) create mode 100644 extras/hs-test/docker/Dockerfile.nginx-server create mode 100644 extras/hs-test/mirroring_test.go create mode 100644 extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf create mode 100644 extras/hs-test/resources/nginx/nginx_server_mirroring.conf create mode 100644 extras/hs-test/suite_nginx_test.go create mode 100644 extras/hs-test/topo-containers/nginxProxyAndServer.yaml create mode 100644 extras/hs-test/topo-network/2taps.yaml diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go index c55fddead54..5def2789528 100644 --- a/extras/hs-test/container.go +++ b/extras/hs-test/container.go @@ -121,12 +121,22 @@ func (c *Container) GetContainerWorkDir() (res string) { return } -func (c *Container) getRunCommand() string { - cmd := "docker run --cap-add=all -d --privileged --network host --rm" - cmd += c.getVolumesAsCliOption() - cmd += c.getEnvVarsAsCliOption() - cmd += " --name " + c.name + " " + c.image + " " + c.extraRunningArgs - return cmd +func (c *Container) getContainerArguments() string { + args := "--cap-add=all --privileged --network host --rm" + args += c.getVolumesAsCliOption() + args += c.getEnvVarsAsCliOption() + args += " --name " + c.name + " " + c.image + return args +} + +func (c *Container) create() { + cmd := "docker create " + c.getContainerArguments() + exechelper.Run(cmd) +} + +func (c *Container) start() { + cmd := "docker start " + c.name + exechelper.Run(cmd) } func (c *Container) run() error { @@ -134,7 +144,8 @@ func (c *Container) run() error { return fmt.Errorf("run container failed: name is blank") } - cmd := c.getRunCommand() + cmd := "docker run -d " + c.getContainerArguments() + " " + c.extraRunningArgs + c.Suite().log(cmd) err := exechelper.Run(cmd) if err != nil { return fmt.Errorf("container run failed: %s", err) @@ -273,6 +284,14 @@ func (c *Container) saveLogs() { f.Close() } +func (c *Container) log() string { + cmd := "docker logs " + c.name + c.Suite().log(cmd) + o, err := exechelper.CombinedOutput(cmd) + c.Suite().assertNil(err) + return string(o) +} + func (c *Container) stop() error { if c.vppInstance != nil && c.vppInstance.apiChannel != nil { c.vppInstance.saveLogs() diff --git a/extras/hs-test/docker/Dockerfile.nginx-server b/extras/hs-test/docker/Dockerfile.nginx-server new file mode 100644 index 00000000000..1971158131b --- /dev/null +++ b/extras/hs-test/docker/Dockerfile.nginx-server @@ -0,0 +1,12 @@ +ARG UBUNTU_VERSION + +FROM ubuntu:${UBUNTU_VERSION} + +RUN apt-get update \ + && apt-get install -y nginx \ + && rm -rf /var/lib/apt/lists/* + +COPY resources/nginx/nginx_server_mirroring.conf /nginx.conf + + +ENTRYPOINT ["nginx", "-c", "/nginx.conf"] diff --git a/extras/hs-test/framework_test.go b/extras/hs-test/framework_test.go index fdce5c45bf7..84aa570e681 100644 --- a/extras/hs-test/framework_test.go +++ b/extras/hs-test/framework_test.go @@ -25,3 +25,8 @@ func TestNoTopo(t *testing.T) { var m NoTopoSuite suite.Run(t, &m) } + +func TestNginx(t *testing.T) { + var m NginxSuite + suite.Run(t, &m) +} diff --git a/extras/hs-test/http_test.go b/extras/hs-test/http_test.go index 96985be1ca7..c1b3b7f2a03 100644 --- a/extras/hs-test/http_test.go +++ b/extras/hs-test/http_test.go @@ -48,8 +48,7 @@ func (s *NoTopoSuite) TestNginxAsServer() { s.assertNil(nginxCont.run()) vpp := s.getContainerByName("vpp").vppInstance - err := vpp.waitForApp("-app", 5) - s.assertNil(err) + vpp.waitForApp("-app", 5) serverAddress := s.netInterfaces[tapInterfaceName].Peer().IP4AddressString() @@ -86,8 +85,7 @@ func runNginxPerf(s *NoTopoSuite, mode, ab_or_wrk string) error { nginxCont := s.getContainerByName("nginx") s.assertNil(nginxCont.run()) - err := vpp.waitForApp("-app", 5) - s.assertNil(err) + vpp.waitForApp("-app", 5) cmd := exec.Command(exeName, args...) s.log(cmd) diff --git a/extras/hs-test/mirroring_test.go b/extras/hs-test/mirroring_test.go new file mode 100644 index 00000000000..97c6c8dbf45 --- /dev/null +++ b/extras/hs-test/mirroring_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/edwarnicke/exechelper" +) + +func (s *NginxSuite) TestMirroring() { + proxyAddress := s.netInterfaces[mirroringClientInterfaceName].Peer().IP4AddressString() + + path := "/64B.json" + + testCommand := "wrk -c 20 -t 10 -d 40 http://" + proxyAddress + ":80" + path + s.log(testCommand) + o, _ := exechelper.Output(testCommand) + s.log(string(o)) + s.assertNotEmpty(o) + + // Check if log output from VPP contains 'no lcl port' warnings + // TODO: Need to change after adding session worker counter + vppProxyContainer := s.getContainerByName(vppProxyContainerName) + logContent := vppProxyContainer.log() + s.assertNotContains(logContent, "no lcl port") +} diff --git a/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf b/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf new file mode 100644 index 00000000000..03af8b76f15 --- /dev/null +++ b/extras/hs-test/resources/nginx/nginx_proxy_mirroring.conf @@ -0,0 +1,82 @@ +user root; +worker_processes 4; +worker_rlimit_nofile 102400; +daemon off; + +events { + use epoll; + worker_connections 102400; + accept_mutex off; +} + +http { + include mime.types; + default_type application/octet-stream; + + access_log off; + + keepalive_timeout 300; + keepalive_requests 1000000; + + proxy_connect_timeout 300; + large_client_header_buffers 4 512k; + client_max_body_size 3000m; + client_header_buffer_size 2048m; + client_body_buffer_size 1024m; + proxy_buffers 16 10240k; + proxy_buffer_size 10240k; + + gzip on; + + upstream bk { + server 10.10.2.1:8091; + keepalive 30000; + } + upstream bk1 { + server 10.10.2.1:8092; + keepalive 30000; + } + upstream bk2 { + server 10.10.2.1:8093; + keepalive 30000; + } + + server { + listen 80; + server_name 10.10.1.2; + + server_tokens off; + + proxy_redirect off; + + location / { + root html; + index index.html index.htm; + proxy_pass http://bk; + proxy_set_header Connection ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header Host $host:$server_port; + chunked_transfer_encoding on; + proxy_http_version 1.1; + mirror /mimic1; + mirror /mimic2; + mirror_request_body on; + } + location /mimic1 { + proxy_pass http://bk1$request_uri; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header Connection ""; + chunked_transfer_encoding on; + proxy_http_version 1.1; + proxy_set_header Host $host:$server_port; + } + location /mimic2 { + proxy_pass http://bk2$request_uri; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header Host $host:$server_port; + proxy_set_header Connection ""; + proxy_http_version 1.1; + chunked_transfer_encoding on; + } + } +} diff --git a/extras/hs-test/resources/nginx/nginx_server_mirroring.conf b/extras/hs-test/resources/nginx/nginx_server_mirroring.conf new file mode 100644 index 00000000000..4056801ea13 --- /dev/null +++ b/extras/hs-test/resources/nginx/nginx_server_mirroring.conf @@ -0,0 +1,32 @@ +master_process on; +worker_rlimit_nofile 10240; +worker_processes 2; +daemon off; + +events { + use epoll; + worker_connections 10240; + accept_mutex off; + multi_accept off; +} + +http { + keepalive_timeout 300s; + keepalive_requests 1000000; + sendfile on; + server { + listen 8091; + listen 8092; + listen 8093; + root /usr/share/nginx; + index index.html index.htm; + location /return_ok + { + return 200 ''; + } + location /64B.json + { + return 200 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'; + } + } +} diff --git a/extras/hs-test/script/build.sh b/extras/hs-test/script/build.sh index 78facd1605d..97278eb4919 100755 --- a/extras/hs-test/script/build.sh +++ b/extras/hs-test/script/build.sh @@ -61,3 +61,4 @@ docker_build () { docker_build hs-test/vpp vpp docker_build hs-test/nginx-ldp nginx +docker_build hs-test/nginx-server nginx-server diff --git a/extras/hs-test/suite_nginx_test.go b/extras/hs-test/suite_nginx_test.go new file mode 100644 index 00000000000..cd67efb6cc5 --- /dev/null +++ b/extras/hs-test/suite_nginx_test.go @@ -0,0 +1,50 @@ +package main + +const ( + // These correspond to names used in yaml config + mirroringClientInterfaceName = "hst_client" + mirroringServerInterfaceName = "hst_server" + vppProxyContainerName = "vpp-proxy" + nginxProxyContainerName = "nginx-proxy" + nginxServerContainerName = "nginx-server" +) + +type NginxSuite struct { + HstSuite +} + +func (s *NginxSuite) SetupSuite() { + s.loadNetworkTopology("2taps") + + s.loadContainerTopology("nginxProxyAndServer") +} + +func (s *NginxSuite) SetupTest() { + s.SetupVolumes() + s.SetupContainers() + + // Setup test conditions + var startupConfig Stanza + startupConfig. + NewStanza("session"). + Append("enable"). + Append("use-app-socket-api").Close() + + // ... for proxy + vppProxyContainer := s.getContainerByName(vppProxyContainerName) + proxyVpp, _ := vppProxyContainer.newVppInstance(startupConfig) + proxyVpp.start() + + clientInterface := s.netInterfaces[mirroringClientInterfaceName] + proxyVpp.createTap(clientInterface, 1) + + serverInterface := s.netInterfaces[mirroringServerInterfaceName] + proxyVpp.createTap(serverInterface, 2) + + nginxContainer := s.getTransientContainerByName(nginxProxyContainerName) + nginxContainer.create() + nginxContainer.copy("./resources/nginx/nginx_proxy_mirroring.conf", "/nginx.conf") + nginxContainer.start() + + proxyVpp.waitForApp("-app", 5) +} diff --git a/extras/hs-test/suite_no_topo_test.go b/extras/hs-test/suite_no_topo_test.go index bca1dbf1828..9051284c780 100644 --- a/extras/hs-test/suite_no_topo_test.go +++ b/extras/hs-test/suite_no_topo_test.go @@ -35,5 +35,5 @@ func (s *NoTopoSuite) SetupTest() { tapInterface := s.netInterfaces[tapInterfaceName] - vpp.createTap(1, tapInterface) + vpp.createTap(tapInterface) } diff --git a/extras/hs-test/topo-containers/nginxProxyAndServer.yaml b/extras/hs-test/topo-containers/nginxProxyAndServer.yaml new file mode 100644 index 00000000000..bac6a2df174 --- /dev/null +++ b/extras/hs-test/topo-containers/nginxProxyAndServer.yaml @@ -0,0 +1,20 @@ +--- +volumes: + - volume: &shared-vol-proxy + host-dir: /tmp/shared-vol-proxy + +containers: + - name: "vpp-proxy" + volumes: + - <<: *shared-vol-proxy + container-dir: "/tmp/vpp" + is-default-work-dir: true + - name: "nginx-proxy" + volumes: + - <<: *shared-vol-proxy + container-dir: "/tmp/nginx" + is-default-work-dir: true + image: "hs-test/nginx-ldp" + is-optional: true + - name: "nginx-server" + image: "hs-test/nginx-server" diff --git a/extras/hs-test/topo-network/2taps.yaml b/extras/hs-test/topo-network/2taps.yaml new file mode 100644 index 00000000000..38f6fca4f31 --- /dev/null +++ b/extras/hs-test/topo-network/2taps.yaml @@ -0,0 +1,18 @@ +--- +devices: + - name: "hst_client" + type: "tap" + ip4: + network: 1 + peer: + name: "" + ip4: + network: 1 + - name: "hst_server" + type: "tap" + ip4: + network: 2 + peer: + name: "" + ip4: + network: 2 diff --git a/extras/hs-test/vppinstance.go b/extras/hs-test/vppinstance.go index 29b86d5adcb..4092d35cfd6 100644 --- a/extras/hs-test/vppinstance.go +++ b/extras/hs-test/vppinstance.go @@ -160,15 +160,16 @@ func (vpp *VppInstance) vppctl(command string, arguments ...any) string { return string(output) } -func (vpp *VppInstance) waitForApp(appName string, timeout int) error { +func (vpp *VppInstance) waitForApp(appName string, timeout int) { for i := 0; i < timeout; i++ { o := vpp.vppctl("show app") if strings.Contains(o, appName) { - return nil + return } time.Sleep(1 * time.Second) } - return fmt.Errorf("timeout while waiting for app '%s'", appName) + vpp.Suite().assertNil(1, "Timeout while waiting for app '%s'", appName) + return } func (vpp *VppInstance) createAfPacket( @@ -253,9 +254,13 @@ func (vpp *VppInstance) addAppNamespace( } func (vpp *VppInstance) createTap( - id uint32, tap *NetInterface, + tapId ...uint32, ) error { + var id uint32 = 1 + if len(tapId) > 0 { + id = tapId[0] + } createTapReq := &tapv2.TapCreateV2{ ID: id, HostIfNameSet: true, -- cgit 1.2.3-korg