aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries/python/HoststackUtil.py
blob: 2c9c080b69ce75bb1815297fce01e69d5331c6c1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
# Copyright (c) 2023 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Host Stack util library."""
import json
from time import sleep
from robot.api import logger

from resources.libraries.python.Constants import Constants
from resources.libraries.python.DUTSetup import DUTSetup
from resources.libraries.python.model.ExportResult import (
    export_hoststack_results
)
from resources.libraries.python.PapiExecutor import PapiSocketExecutor
from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error

class HoststackUtil():
    """Utilities for Host Stack tests."""

    @staticmethod
    def get_vpp_echo_command(vpp_echo_attributes):
        """Construct the vpp_echo command using the specified attributes.

        :param vpp_echo_attributes: vpp_echo test program attributes.
        :type vpp_echo_attributes: dict
        :returns: Command line components of the vpp_echo command
            'name' - program name
            'args' - command arguments.
        :rtype: dict
        """
        proto = vpp_echo_attributes[u"uri_protocol"]
        addr = vpp_echo_attributes[u"uri_ip4_addr"]
        port = vpp_echo_attributes[u"uri_port"]
        vpp_echo_cmd = {}
        vpp_echo_cmd[u"name"] = u"vpp_echo"
        vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
            f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
            f"{vpp_echo_attributes[u'json_output']} " \
            f"uri {proto}://{addr}/{port} " \
            f"nthreads {vpp_echo_attributes[u'nthreads']} " \
            f"mq-size {vpp_echo_attributes[u'mq_size']} " \
            f"nclients {vpp_echo_attributes[u'nclients']} " \
            f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
            f"time {vpp_echo_attributes[u'time']} " \
            f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
            f"TX={vpp_echo_attributes[u'tx_bytes']} " \
            f"RX={vpp_echo_attributes[u'rx_bytes']}"
        if vpp_echo_attributes[u"rx_results_diff"]:
            vpp_echo_cmd[u"args"] += u" rx-results-diff"
        if vpp_echo_attributes[u"tx_results_diff"]:
            vpp_echo_cmd[u"args"] += u" tx-results-diff"
        if vpp_echo_attributes[u"use_app_socket_api"]:
            vpp_echo_cmd[u"args"] += u" use-app-socket-api"
        return vpp_echo_cmd

    @staticmethod
    def get_iperf3_command(iperf3_attributes):
        """Construct the iperf3 command using the specified attributes.

        :param iperf3_attributes: iperf3 test program attributes.
        :type iperf3_attributes: dict
        :returns: Command line components of the iperf3 command
            'env_vars' - environment variables
            'name' - program name
            'args' - command arguments.
        :rtype: dict
        """
        iperf3_cmd = {}
        iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
            f"{Constants.RESOURCES_TPL_VCL}/" \
            f"{iperf3_attributes[u'vcl_config']}"
        if iperf3_attributes[u"ld_preload"]:
            iperf3_cmd[u"env_vars"] += \
                f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
        if iperf3_attributes[u'transparent_tls']:
            iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"

        json_results = u" --json" if iperf3_attributes[u'json'] else u""
        ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
                     in iperf3_attributes else u""
        iperf3_cmd[u"name"] = u"iperf3"
        iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
                              f"--interval 0{json_results} " \
                              f"--version{iperf3_attributes[u'ip_version']}"

        if iperf3_attributes[u"role"] == u"server":
            iperf3_cmd[u"args"] += u" --one-off"
        else:
            iperf3_cmd[u"args"] += u" --get-server-output"
            if u"parallel" in iperf3_attributes:
                iperf3_cmd[u"args"] += \
                    f" --parallel {iperf3_attributes[u'parallel']}"
            if u"time" in iperf3_attributes:
                iperf3_cmd[u"args"] += \
                    f" --time {iperf3_attributes[u'time']}"
            if iperf3_attributes[u"udp"]:
                iperf3_cmd[u"args"] += u" --udp"
                iperf3_cmd[u"args"] += \
                    f" --bandwidth {iperf3_attributes[u'bandwidth']}"
            if iperf3_attributes[u"length"] > 0:
                iperf3_cmd[u"args"] += \
                    f" --length {iperf3_attributes[u'length']}"
        return iperf3_cmd

    @staticmethod
    def set_hoststack_quic_fifo_size(node, fifo_size):
        """Set the QUIC protocol fifo size.

        :param node: Node to set the QUIC fifo size on.
        :param fifo_size: fifo size, passed to the quic set fifo-size command.
        :type node: dict
        :type fifo_size: str
        """
        cmd = f"quic set fifo-size {fifo_size}"
        PapiSocketExecutor.run_cli_cmd(node, cmd)

    @staticmethod
    def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
                                         fail_on_error=False):
        """Set the Hoststack QUIC crypto engine on node

        :param node: Node to enable/disable HostStack.
        :param quic_crypto_engine: type of crypto engine
        :type node: dict
        :type quic_crypto_engine: str
        """
        vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
        if quic_crypto_engine == u"nocrypto":
            logger.trace(u"No QUIC crypto engine.")
            return

        if quic_crypto_engine in vpp_crypto_engines:
            cmds = [u"quic set crypto api vpp",
                    f"set crypto handler aes-128-gcm {quic_crypto_engine}",
                    f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
        elif quic_crypto_engine == u"picotls":
            cmds = [u"quic set crypto api picotls"]
        else:
            raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")

        for cmd in cmds:
            try:
                PapiSocketExecutor.run_cli_cmd(node, cmd)
            except AssertionError:
                if fail_on_error:
                    raise

    @staticmethod
    def _get_hoststack_test_program_logs(node, program_name):
        """Get HostStack test program stdout log.

        :param node: DUT node.
        :param program_name: test program.
        :type node: dict
        :type program_name: str
        """
        cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
        stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
            message=f"Get {program_name} stdout log failed!")

        cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
        stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
            message=f"Get {program_name} stderr log failed!")

        return stdout_log, stderr_log

    @staticmethod
    def get_hoststack_test_program_logs(node, program):
        """Get HostStack test program stdout log.

        :param node: DUT node.
        :param program: test program.
        :type node: dict
        :type program: dict
        """
        program_name = program[u"name"]
        program_stdout_log, program_stderr_log = \
            HoststackUtil._get_hoststack_test_program_logs(node,
                                                           program_name)
        if len(program_stdout_log) == 0 and len(program_stderr_log) == 0:
            logger.trace(f"Retrying {program_name} log retrieval")
            program_stdout_log, program_stderr_log = \
               HoststackUtil._get_hoststack_test_program_logs(node,
                                                              program_name)
        return program_stdout_log, program_stderr_log

    @staticmethod
    def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
        """Construct the NGINX command using the specified attributes.

        :param nginx_attributes: NGINX test program attributes.
        :param nginx_version: NGINX version.
        :param nginx_ins_dir: NGINX install dir.
        :type nginx_attributes: dict
        :type nginx_version: str
        :type nginx_ins_dir: str
        :returns: Command line components of the NGINX command
            'env_vars' - environment variables
            'name' - program name
            'args' - command arguments.
            'path' - program path.
        :rtype: dict
        """
        nginx_cmd = dict()
        nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
                                 f"{Constants.RESOURCES_TPL_VCL}/" \
                                 f"{nginx_attributes[u'vcl_config']}"
        if nginx_attributes[u"ld_preload"]:
            nginx_cmd[u"env_vars"] += \
                f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
        if nginx_attributes[u'transparent_tls']:
            nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"

        nginx_cmd[u"name"] = u"nginx"
        nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
        nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
                             f"nginx-{nginx_version}/conf/nginx.conf"
        return nginx_cmd

    @staticmethod
    def start_hoststack_test_program(node, namespace, core_list, program):
        """Start the specified HostStack test program.

        :param node: DUT node.
        :param namespace: Net Namespace to run program in.
        :param core_list: List of cpu's to pass to taskset to pin the test
            program to a different set of cores on the same numa node as VPP.
        :param program: Test program.
        :type node: dict
        :type namespace: str
        :type core_list: str
        :type program: dict
        :returns: Process ID
        :rtype: int
        :raises RuntimeError: If node subtype is not a DUT or startup failed.
        """
        if node[u"type"] != u"DUT":
            raise RuntimeError(u"Node type is not a DUT!")

        program_name = program[u"name"]
        DUTSetup.kill_program(node, program_name, namespace)

        if namespace == u"default":
            shell_cmd = u"sh -c"
        else:
            shell_cmd = f"ip netns exec {namespace} sh -c"

        env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
        args = program[u"args"]
        program_path = program.get(u"path", u"")
        # NGINX used `worker_cpu_affinity` in configuration file
        taskset_cmd = u"" if program_name == u"nginx" else \
                                             f"taskset --cpu-list {core_list}"
        cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
              f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
              f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
        try:
            exec_cmd_no_error(node, cmd, sudo=True)
            return DUTSetup.get_pid(node, program_name)[0]
        except RuntimeError:
            stdout_log, stderr_log = \
                HoststackUtil.get_hoststack_test_program_logs(node,
                                                              program)
            raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
                               f"{stderr_log}\nSTDOUT:\n{stdout_log}")
        return None

    @staticmethod
    def stop_hoststack_test_program(node, program, pid):
        """Stop the specified Hoststack test program.

        :param node: DUT node.
        :param program: Test program.
        :param pid: Process ID of test program.
        :type node: dict
        :type program: dict
        :type pid: int
        """
        program_name = program[u"name"]
        if program_name == u"nginx":
            cmd = u"nginx -s quit"
            errmsg = u"Quit nginx failed!"
        else:
            cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
                f'then kill -s SIGTERM {pid}; fi'
            errmsg = f"Kill {program_name} ({pid}) failed!"

        exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)

    @staticmethod
    def hoststack_test_program_finished(node, program_pid, program,
                                        other_node, other_program):
        """Wait for the specified HostStack test program process to complete.

        :param node: DUT node.
        :param program_pid: test program pid.
        :param program: test program
        :param other_node: DUT node of other hoststack program
        :param other_program: other test program
        :type node: dict
        :type program_pid: str
        :type program: dict
        :type other_node: dict
        :type other_program: dict
        :raises RuntimeError: If node subtype is not a DUT.
        """
        if node[u"type"] != u"DUT":
            raise RuntimeError(u"Node type is not a DUT!")
        if other_node[u"type"] != u"DUT":
            raise RuntimeError(u"Other node type is not a DUT!")

        cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
        try:
            exec_cmd(node, cmd, sudo=True)
        except:
            sleep(180)
            if u"client" in program[u"args"]:
                role = u"client"
            else:
                role = u"server"
            program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(node, program)
            if len(program_stdout) > 0:
                logger.debug(f"{program[u'name']} {role} stdout log:\n"
                             f"{program_stdout}")
            else:
                logger.debug(f"Empty {program[u'name']} {role} stdout log :(")
            if len(program_stderr) > 0:
                logger.debug(f"{program[u'name']} stderr log:\n"
                             f"{program_stderr}")
            else:
                logger.debug(f"Empty {program[u'name']} stderr log :(")
            if u"client" in other_program[u"args"]:
                role = u"client"
            else:
                role = u"server"
            program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(other_node,
                                                              other_program)
            if len(program_stdout) > 0:
                logger.debug(f"{other_program[u'name']} {role} stdout log:\n"
                             f"{program_stdout}")
            else:
                logger.debug(f"Empty {other_program[u'name']} "
                             f"{role} stdout log :(")
            if len(program_stderr) > 0:
                logger.debug(f"{other_program[u'name']} {role} stderr log:\n"
                             f"{program_stderr}")
            else:
                logger.debug(f"Empty {other_program[u'name']} "
                             f"{role} stderr log :(")
            raise
        # Wait a bit for stdout/stderr to be flushed to log files
        sleep(1)

    @staticmethod
    def analyze_hoststack_test_program_output(
            node, role, nsim_attr, program):
        """Gather HostStack test program output and check for errors.

        The [defer_fail] return bool is used instead of failing immediately
        to allow the analysis of both the client and server instances of
        the test program for debugging a test failure.  When [defer_fail]
        is true, then the string returned is debug output instead of
        JSON formatted test program results.

        :param node: DUT node.
        :param role: Role (client|server) of test program.
        :param nsim_attr: Network Simulation Attributes.
        :param program: Test program.
        :param program_args: List of test program args.
        :type node: dict
        :type role: str
        :type nsim_attr: dict
        :type program: dict
        :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
            test program output or failure debug output.
        :rtype: bool, str
        :raises RuntimeError: If node subtype is not a DUT.
        """
        if node[u"type"] != u"DUT":
            raise RuntimeError(u"Node type is not a DUT!")

        program_name = program[u"name"]
        program_stdout, program_stderr = \
            HoststackUtil.get_hoststack_test_program_logs(node, program)

        env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
        program_cmd = f"{env_vars}{program_name} {program[u'args']}"
        test_results = f"Test Results of '{program_cmd}':\n"

        if nsim_attr[u"output_nsim_enable"] or \
            nsim_attr[u"xc_nsim_enable"]:
            if nsim_attr[u"output_nsim_enable"]:
                feature_name = u"output"
            else:
                feature_name = u"cross-connect"
            test_results += \
                f"NSIM({feature_name}): delay " \
                f"{nsim_attr[u'delay_in_usec']} usecs, " \
                f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
                f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
                f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
                f"pkts/drop\n"

        test_results += \
            f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
            f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"

        if u"error" in program_stderr.lower():
            test_results += f"ERROR DETECTED:\n{program_stderr}"
            return (True, test_results)
        if not program_stdout:
            test_results += f"\nNo {program} test data retrieved!\n"
            ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
                                             sudo=True)
            test_results += f"{ls_stdout}\n"
            return (True, test_results)
        if program[u"name"] == u"vpp_echo":
            if u"JSON stats" in program_stdout and \
                    u'"has_failed": "0"' in program_stdout:
                json_start = program_stdout.find(u"{")
                json_end = program_stdout.find(u',\n  "closing"')
                json_results = f"{program_stdout[json_start:json_end]}\n}}"
                program_json = json.loads(json_results)
                export_hoststack_results(
                    bandwidth=program_json["rx_bits_per_second"],
                    duration=program_json["time"]
                )
            else:
                test_results += u"Invalid test data output!\n" + program_stdout
                return (True, test_results)
        elif program[u"name"] == u"iperf3":
            test_results += program_stdout
            program_json = json.loads(program_stdout)[u"intervals"][0][u"sum"]
            try:
                retransmits = program_json["retransmits"]
            except(KeyError):
                retransmits = None
            export_hoststack_results(
                bandwidth=program_json["bits_per_second"],
                duration=program_json["seconds"],
                retransmits=retransmits
            )
        else:
            test_results += u"Unknown HostStack Test Program!\n" + \
                            program_stdout
            return (True, program_stdout)
        return (False, json.dumps(program_json))

    @staticmethod
    def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
        """Return True if either HostStack test program fail was deferred.

        :param server_defer_fail: server no results value.
        :param client_defer_fail: client no results value.
        :type server_defer_fail: bool
        :type client_defer_fail: bool
        :rtype: bool
        """
        return server_defer_fail and client_defer_fail

    @staticmethod
    def log_vpp_hoststack_data(node):
        """Retrieve and log VPP HostStack data.

        :param node: DUT node.
        :type node: dict
        :raises RuntimeError: If node subtype is not a DUT or startup failed.
        """

        if node[u"type"] != u"DUT":
            raise RuntimeError(u"Node type is not a DUT!")

        PapiSocketExecutor.run_cli_cmd(node, u"show error")
        PapiSocketExecutor.run_cli_cmd(node, u"show interface")