From 9a261ea61549fc6a5c23369d2e236b002dc35038 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Fri, 22 Sep 2017 14:59:40 +0200 Subject: CSIT-748 vnf-agent integration CSIT-773 Implement RF keywords for k8s - Implementation of Test Suite Setup for Ligato vnf-agent testing - Implementation of KubernetesUtil for controlling kubectl - Yaml templates for L2XC topology with 1cswitch and 1cnf - Yaml templates for L2BD topology with 1cswitch and 1cnf - ligato bootstrap script for creating vnf-agent image Change-Id: Iebefde0eb984a27a0afcdf29fe549ca4edf8a61e Signed-off-by: Peter Mikus --- resources/libraries/python/ContainerUtils.py | 117 ++++--- resources/libraries/python/InterfaceUtil.py | 41 +++ resources/libraries/python/KubernetesUtils.py | 372 +++++++++++++++++++++++ resources/libraries/python/VppConfigGenerator.py | 91 +++--- resources/libraries/python/constants.py | 3 + 5 files changed, 523 insertions(+), 101 deletions(-) create mode 100644 resources/libraries/python/KubernetesUtils.py (limited to 'resources/libraries/python') diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index fb2695fe44..b56fb0dc24 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -12,7 +12,7 @@ # limitations under the License. # Bug workaround in pylint for abstract classes. -#pylint: disable=W0223 +# pylint: disable=W0223 """Library to manipulate Containers.""" @@ -93,11 +93,12 @@ class ContainerManager(object): def construct_containers(self, **kwargs): """Construct 1..N container(s) on node with specified name. + Ordinal number is automatically added to the name of container as suffix. - :param kwargs: Name of container. - :param kwargs: str + :param kwargs: Named parameters. + :param kwargs: dict """ name = kwargs['name'] for i in range(kwargs['count']): @@ -311,7 +312,6 @@ class ContainerEngine(object): # Create config instance vpp_config = VppConfigGenerator() vpp_config.set_node(self.container.node) - vpp_config.set_config_filename(config_filename) vpp_config.add_unix_cli_listen() vpp_config.add_unix_nodaemon() vpp_config.add_unix_exec('/tmp/running.exec') @@ -326,15 +326,15 @@ class ContainerEngine(object): self.execute('mkdir -p /etc/vpp/') self.execute('echo "{c}" | tee {f}' .format(c=vpp_config.get_config_str(), - f=vpp_config.get_config_filename())) + f=config_filename)) - def create_vpp_exec_config(self, vat_template_file, **args): + def create_vpp_exec_config(self, vat_template_file, **kwargs): """Create VPP exec configuration on container. :param vat_template_file: File name of a VAT template script. - :param args: Parameters for VAT script. + :param kwargs: Parameters for VAT script. :type vat_template_file: str - :type args: dict + :type kwargs: dict """ vat_file_path = '{p}/{f}'.format(p=Constants.RESOURCES_TPL_VAT, f=vat_template_file) @@ -342,7 +342,7 @@ class ContainerEngine(object): with open(vat_file_path, 'r') as template_file: cmd_template = template_file.readlines() for line_tmpl in cmd_template: - vat_cmd = line_tmpl.format(**args) + vat_cmd = line_tmpl.format(**kwargs) self.execute('echo "{c}" >> /tmp/running.exec' .format(c=vat_cmd.replace('\n', ''))) @@ -354,6 +354,28 @@ class ContainerEngine(object): """Check if container is present.""" raise NotImplementedError + def _configure_cgroup(self, name): + """Configure the control group associated with a container. + + :param name: Name of cgroup. + :type name: str + :raises RuntimeError: If applying cgroup settings via cgset failed. + """ + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgcreate -g cpuset:/{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to copy cgroup settings from root.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.cpu_exclusive=0 /{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + + ret, _, _ = self.container.ssh.exec_command_sudo( + 'cgset -r cpuset.mem_exclusive=0 /{name}'.format(name=name)) + if int(ret) != 0: + raise RuntimeError('Failed to apply cgroup settings.') + class LXC(ContainerEngine): """LXC implementation.""" @@ -363,8 +385,7 @@ class LXC(ContainerEngine): super(LXC, self).__init__() def acquire(self, force=True): - """Acquire a privileged system object where configuration is stored and - where user information can be stored. + """Acquire a privileged system object where configuration is stored. :param force: If a container exists, destroy it and create a new container. @@ -398,6 +419,7 @@ class LXC(ContainerEngine): if int(ret) != 0: raise RuntimeError('Failed to write {c.name} config.' .format(c=self.container)) + self._configure_cgroup('lxc') def create(self): """Create/deploy an application inside a container on system. @@ -415,13 +437,25 @@ class LXC(ContainerEngine): raise RuntimeError('Failed to start container {c.name}.' .format(c=self.container)) self._lxc_wait('RUNNING') - self._lxc_cgroup(state_object='cpuset.cpus', - value=cpuset_cpus) + + # Workaround for LXC to be able to allocate all cpus including isolated. + cmd = 'cgset --copy-from / lxc/' + ret, _, _ = self.container.ssh.exec_command_sudo(cmd) + if int(ret) != 0: + raise RuntimeError('Failed to copy cgroup to LXC') + + cmd = 'lxc-cgroup --name {c.name} cpuset.cpus {cpus}'\ + .format(c=self.container, cpus=cpuset_cpus) + ret, _, _ = self.container.ssh.exec_command_sudo(cmd) + if int(ret) != 0: + raise RuntimeError('Failed to set cpuset.cpus to container ' + '{c.name}.'.format(c=self.container)) def execute(self, command): - """Start a process inside a running container. Runs the specified - command inside the container specified by name. The container has to - be running already. + """Start a process inside a running container. + + Runs the specified command inside the container specified by name. The + container has to be running already. :param command: Command to run inside container. :type command: str @@ -530,33 +564,6 @@ class LXC(ContainerEngine): raise RuntimeError('Failed to wait for state "{s}" of container ' '{c.name}.'.format(s=state, c=self.container)) - def _lxc_cgroup(self, state_object, value=''): - """Manage the control group associated with a container. - - :param state_object: Specify the state object name. - :param value: Specify the value to assign to the state object. If empty, - then action is GET, otherwise is action SET. - :type state_object: str - :type value: str - :raises RuntimeError: If getting/setting state of a container failed. - """ - cmd = 'lxc-cgroup --name {c.name} {s} {v}'\ - .format(c=self.container, s=state_object, v=value) - - ret, _, _ = self.container.ssh.exec_command_sudo( - 'cgset --copy-from / lxc') - if int(ret) != 0: - raise RuntimeError('Failed to copy cgroup settings from root.') - - ret, _, _ = self.container.ssh.exec_command_sudo(cmd) - if int(ret) != 0: - if value: - raise RuntimeError('Failed to set {s} of container {c.name}.' - .format(s=state_object, c=self.container)) - else: - raise RuntimeError('Failed to get {s} of container {c.name}.' - .format(s=state_object, c=self.container)) - class Docker(ContainerEngine): """Docker implementation.""" @@ -584,6 +591,7 @@ class Docker(ContainerEngine): if int(ret) != 0: raise RuntimeError('Failed to create container {c.name}.' .format(c=self.container)) + self._configure_cgroup('docker') def create(self): """Create/deploy container. @@ -613,7 +621,7 @@ class Docker(ContainerEngine): cmd = 'docker run '\ '--privileged --detach --interactive --tty --rm '\ - '--cgroup-parent lxc {cpuset_cpus} {cpuset_mems} {publish} '\ + '--cgroup-parent docker {cpuset_cpus} {cpuset_mems} {publish} '\ '{env} {volume} --name {container.name} {container.image} '\ '{command}'.format(cpuset_cpus=cpuset_cpus, cpuset_mems=cpuset_mems, container=self.container, command=command, @@ -627,9 +635,10 @@ class Docker(ContainerEngine): self.info() def execute(self, command): - """Start a process inside a running container. Runs the specified - command inside the container specified by name. The container has to - be running already. + """Start a process inside a running container. + + Runs the specified command inside the container specified by name. The + container has to be running already. :param command: Command to run inside container. :type command: str @@ -731,12 +740,26 @@ class Container(object): pass def __getattr__(self, attr): + """Get attribute custom implementation. + + :param attr: Attribute to get. + :type attr: str + :returns: Attribute value or None. + :rtype: any + """ try: return self.__dict__[attr] except KeyError: return None def __setattr__(self, attr, value): + """Set attribute custom implementation. + + :param attr: Attribute to set. + :param value: Value to set. + :type attr: str + :type value: any + """ try: # Check if attribute exists self.__dict__[attr] diff --git a/resources/libraries/python/InterfaceUtil.py b/resources/libraries/python/InterfaceUtil.py index 71d36c1c9d..795bb52933 100644 --- a/resources/libraries/python/InterfaceUtil.py +++ b/resources/libraries/python/InterfaceUtil.py @@ -449,6 +449,47 @@ class InterfaceUtil(object): VatJsonUtil.update_vpp_interface_data_from_json(node, interface_dump_json) + @staticmethod + def update_nic_interface_names(node): + """Update interface names based on nic type and PCI address. + + This method updates interface names in the same format as VPP does. + + :param node: Node dictionary. + :type node: dict + """ + for ifc in node['interfaces'].values(): + if_pci = ifc['pci_address'].replace('.', ':').split(':') + bus = '{:x}'.format(int(if_pci[1], 16)) + dev = '{:x}'.format(int(if_pci[2], 16)) + fun = '{:x}'.format(int(if_pci[3], 16)) + loc = '{bus}/{dev}/{fun}'.format(bus=bus, dev=dev, fun=fun) + if ifc['model'] == 'Intel-XL710': + ifc['name'] = 'FortyGigabitEthernet{loc}'.format(loc=loc) + elif ifc['model'] == 'Intel-X710': + ifc['name'] = 'TenGigabitEthernet{loc}'.format(loc=loc) + elif ifc['model'] == 'Intel-X520-DA2': + ifc['name'] = 'TenGigabitEthernet{loc}'.format(loc=loc) + elif ifc['model'] == 'Cisco-VIC-1385': + ifc['name'] = 'FortyGigabitEthernet{loc}'.format(loc=loc) + elif ifc['model'] == 'Cisco-VIC-1227': + ifc['name'] = 'TenGigabitEthernet{loc}'.format(loc=loc) + else: + ifc['name'] = 'UnknownEthernet{loc}'.format(loc=loc) + + @staticmethod + def update_nic_interface_names_on_all_duts(nodes): + """Update interface names based on nic type and PCI address on all DUTs. + + This method updates interface names in the same format as VPP does. + + :param nodes: Topology nodes. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + InterfaceUtil.update_nic_interface_names(node) + @staticmethod def update_tg_interface_data_on_node(node): """Update interface name for TG/linux node in DICT__nodes. diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py new file mode 100644 index 0000000000..5faa056ddc --- /dev/null +++ b/resources/libraries/python/KubernetesUtils.py @@ -0,0 +1,372 @@ +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library to control Kubernetes kubectl.""" + +import time +import yaml + +from resources.libraries.python.constants import Constants +from resources.libraries.python.topology import NodeType +from resources.libraries.python.ssh import SSH +from resources.libraries.python.CpuUtils import CpuUtils +from resources.libraries.python.VppConfigGenerator import VppConfigGenerator + +__all__ = ["KubernetesUtils"] + + +class KubernetesUtils(object): + """Kubernetes utilities class.""" + + def __init__(self): + """Initialize KubernetesUtils class.""" + pass + + @staticmethod + def setup_kubernetes_on_node(node): + """Set up Kubernetes on node. + + :param node: DUT node. + :type node: dict + :raises RuntimeError: If Kubernetes setup failed on node. + """ + ssh = SSH() + ssh.connect(node) + + cmd = '{dir}/{lib}/k8s_setup.sh '.format(dir=Constants.REMOTE_FW_DIR, + lib=Constants.RESOURCES_LIB_SH) + (ret_code, _, _) = ssh.exec_command(cmd, timeout=120) + if int(ret_code) != 0: + raise RuntimeError('Failed to setup Kubernetes on {node}.' + .format(node=node['host'])) + + @staticmethod + def setup_kubernetes_on_all_duts(nodes): + """Set up Kubernetes on all DUTs. + + :param nodes: Topology nodes. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + KubernetesUtils.setup_kubernetes_on_node(node) + + @staticmethod + def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs): + """Apply Kubernetes resource on node. + + :param node: DUT node. + :param yaml_file: YAML configuration file. + :param kwargs: Key-value pairs to replace in YAML template. + :type node: dict + :type yaml_file: str + :type kwargs: dict + :raises RuntimeError: If applying Kubernetes template failed. + """ + ssh = SSH() + ssh.connect(node) + + stream = file('{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S, + yaml=yaml_file), 'r') + + for data in yaml.load_all(stream): + data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(), + yaml.dump(data, default_flow_style=False)) + # Workaround to avoid using RAW string anotated with | in YAML as + # library + bash is misinterpreting spaces. + data = data.replace('.conf:\n', '.conf: |\n') + cmd = 'cat < /path/to/file") does not - # work on most platforms... (ret, _, _) = \ - ssh.exec_command('echo "{0}" | sudo tee {1}'. - format(self._vpp_config, - self._vpp_config_filename)) + ssh.exec_command('echo "{config}" | sudo tee {filename}'. + format(config=self._vpp_config, + filename=filename)) if ret != 0: raise RuntimeError('Writing config file failed to node {}'. format(self._hostname)) - # Instead of restarting, we'll do separate start and stop - # actions. This way we don't care whether VPP was running - # to begin with. - ssh.exec_command('sudo service {} stop' - .format(self._vpp_service_name)) - (ret, _, _) = \ - ssh.exec_command('sudo service {} start' + if restart_vpp: + # Instead of restarting, we'll do separate start and stop + # actions. This way we don't care whether VPP was running + # to begin with. + ssh.exec_command('sudo service {} stop' .format(self._vpp_service_name)) - if ret != 0: - raise RuntimeError('Restarting VPP failed on node {}'. - format(self._hostname)) - - # Sleep seconds, up to times, - # and verify if VPP is running. - for _ in range(retries): - time.sleep(waittime) (ret, _, _) = \ - ssh.exec_command('echo show hardware-interfaces | ' - 'nc 0 5002 || echo "VPP not yet running"') - if ret == 0: - break - else: - raise RuntimeError('VPP failed to restart on node {}'. - format(self._hostname)) + ssh.exec_command('sudo service {} start' + .format(self._vpp_service_name)) + if ret != 0: + raise RuntimeError('Restarting VPP failed on node {}'. + format(self._hostname)) + + # Sleep seconds, up to times, + # and verify if VPP is running. + for _ in range(retries): + time.sleep(waittime) + (ret, _, _) = \ + ssh.exec_command('echo show hardware-interfaces | ' + 'nc 0 5002 || echo "VPP not yet running"') + if ret == 0: + break + else: + raise RuntimeError('VPP failed to restart on node {}'. + format(self._hostname)) diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py index a8d40a2a26..01a96a861b 100644 --- a/resources/libraries/python/constants.py +++ b/resources/libraries/python/constants.py @@ -35,6 +35,9 @@ class Constants(object): # QEMU install directory QEMU_INSTALL_DIR = '/opt/qemu-2.5.0' + # Kubernetes templates location + RESOURCES_TPL_K8S = 'resources/templates/kubernetes' + # Honeycomb directory location at topology nodes: REMOTE_HC_DIR = '/opt/honeycomb' -- cgit 1.2.3-korg