Redhat and small system support

Initial 17.10 commit
Final 17.07 cleanup, 17.10 next
Added CentOS grub support, this should complete the CentOS support
Added Centos install/unistall
Added TCP parameters.

Change-Id: I064e3a4118969ac36e62924a6a3f8a98f132ba60
Signed-off-by: John DeNisco <jdenisco@cisco.com>
Signed-off-by: Dave Barach <dave@barachs.net>

1 files changed, 662 insertions, 0 deletions

diff --git a/extras/vpp_config/vpplib/VPPUtil.py b/extras/vpp_config/vpplib/VPPUtil.py
new file mode 100644
index 00000000000..350b7759a03
--- /dev/null
+++ b/extras/vpp_config/vpplib/VPPUtil.py
@@ -0,0 +1,662 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""VPP util library"""
+import logging
+import re
+import subprocess
+import platform
+
+from collections import Counter
+
+# VPP_VERSION = '1707'
+VPP_VERSION = '1710'
+
+
+class VPPUtil(object):
+    """General class for any VPP related methods/functions."""
+
+    @staticmethod
+    def exec_command(cmd, timeout=None):
+        """Execute a command on the local node.
+
+        :param cmd: Command to run locally.
+        :param timeout: Timeout value
+        :type cmd: str
+        :type timeout: int
+        :return return_code, stdout, stderr
+        :rtype: tuple(int, str, str)
+        """
+
+        logging.info(" Local Command: {}".format(cmd))
+        out = ''
+        err = ''
+        prc = subprocess.Popen(cmd, shell=True, bufsize=1,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+
+        with prc.stdout:
+            for line in iter(prc.stdout.readline, b''):
+                logging.info("  {}".format(line.strip('\n')))
+                out += line
+
+        with prc.stderr:
+            for line in iter(prc.stderr.readline, b''):
+                logging.warn("  {}".format(line.strip('\n')))
+                err += line
+
+        ret = prc.wait()
+
+        return ret, out, err
+
+    def _autoconfig_backup_file(self, filename):
+        """
+        Create a backup file.
+
+        :param filename: The file to backup
+        :type filename: str
+        """
+
+        # Does a copy of the file exist, if not create one
+        ofile = filename + '.orig'
+        (ret, stdout, stderr) = self.exec_command('ls {}'.format(ofile))
+        if ret != 0:
+            logging.debug(stderr)
+            if stdout.strip('\n') != ofile:
+                cmd = 'sudo cp {} {}'.format(filename, ofile)
+                (ret, stdout, stderr) = self.exec_command(cmd)
+                if ret != 0:
+                    logging.debug(stderr)
+
+    def _install_vpp_pkg_ubuntu(self, node, pkg):
+        """
+        Install the VPP packages
+
+        :param node: Node dictionary
+        :param pkg: The vpp packages
+        :type node: dict
+        :type pkg: string
+        """
+
+        cmd = 'apt-get -y install {}'.format(pkg)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.format(
+                cmd, node['host'], stdout, stderr))
+
+    def _install_vpp_pkg_centos(self, node, pkg):
+        """
+        Install the VPP packages
+
+        :param node: Node dictionary
+        :param pkg: The vpp packages
+        :type node: dict
+        :type pkg: string
+        """
+
+        cmd = 'yum -y install {}'.format(pkg)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.format(
+                cmd, node['host'], stdout, stderr))
+
+    def _install_vpp_ubuntu(self, node, fdio_release=VPP_VERSION,
+                            ubuntu_version='xenial'):
+        """
+        Install the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :param fdio_release: VPP release number
+        :param ubuntu_version: Ubuntu Version
+        :type node: dict
+        :type fdio_release: string
+        :type ubuntu_version: string
+        """
+
+        # Modify the sources list
+        sfile = '/etc/apt/sources.list.d/99fd.io.list'
+
+        # Backup the sources list
+        self._autoconfig_backup_file(sfile)
+
+        # Remove the current file
+        cmd = 'rm {}'.format(sfile)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            logging.debug('{} failed on node {} {}'.format(
+                cmd,
+                node['host'],
+                stderr))
+
+        reps = 'deb [trusted=yes] https://nexus.fd.io/content/'
+        reps += 'repositories/fd.io.stable.{}.ubuntu.{}.main/ ./\n' \
+            .format(fdio_release, ubuntu_version)
+
+        cmd = 'echo "{0}" | sudo tee {1}'.format(reps, sfile)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {}'.format(
+                cmd,
+                node['host'],
+                stderr))
+
+        # Install the package
+        cmd = 'apt-get -y update'
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} apt-get update failed on node {} {}'.format(
+                cmd,
+                node['host'],
+                stderr))
+
+        self._install_vpp_pkg_ubuntu(node, 'vpp-lib')
+        self._install_vpp_pkg_ubuntu(node, 'vpp')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-plugins')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-dpdk-dkms')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-dpdk-dev')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-api-python')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-api-java')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-api-lua')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-dev')
+        self._install_vpp_pkg_ubuntu(node, 'vpp-dbg')
+
+    def _install_vpp_centos(self, node, fdio_release=VPP_VERSION,
+                            centos_version='centos7'):
+        """
+        Install the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :param fdio_release: VPP release number
+        :param centos_version: Ubuntu Version
+        :type node: dict
+        :type fdio_release: string
+        :type centos_version: string
+        """
+
+        # Modify the sources list
+        sfile = '/etc/yum.repos.d/fdio-release.repo'
+
+        # Backup the sources list
+        self._autoconfig_backup_file(sfile)
+
+        # Remove the current file
+        cmd = 'rm {}'.format(sfile)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            logging.debug('{} failed on node {} {}'.format(
+                cmd,
+                node['host'],
+                stderr))
+
+        reps = '[fdio-stable-{}]\n'.format(fdio_release)
+        reps += 'name=fd.io stable/{} branch latest merge\n'.format(fdio_release)
+        reps += 'baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.{}.{}/\n'.\
+            format(fdio_release, centos_version)
+        reps += 'enabled=1\n'
+        reps += 'gpgcheck=0'
+
+        cmd = 'echo "{0}" | sudo tee {1}'.format(reps, sfile)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {}'.format(
+                cmd,
+                node['host'],
+                stderr))
+
+        # Install the packages
+ 
+        self._install_vpp_pkg_centos(node, 'vpp-lib')
+        self._install_vpp_pkg_centos(node, 'vpp')
+        self._install_vpp_pkg_centos(node, 'vpp-plugins')
+        # jadfix Check with Ole
+        # self._install_vpp_pkg_centos(node, 'vpp-dpdk-devel')
+        self._install_vpp_pkg_centos(node, 'vpp-api-python')
+        self._install_vpp_pkg_centos(node, 'vpp-api-java')
+        self._install_vpp_pkg_centos(node, 'vpp-api-lua')
+        self._install_vpp_pkg_centos(node, 'vpp-devel')
+
+    def install_vpp(self, node):
+        """
+        Install the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :type node: dict
+        """
+        distro = self.get_linux_distro()
+        if distro[0] == 'Ubuntu':
+            self._install_vpp_ubuntu(node)
+        elif distro[0] == 'CentOS Linux':
+            logging.info("Install CentOS")
+            self._install_vpp_centos(node)
+        else:
+            return
+
+    def _uninstall_vpp_pkg_ubuntu(self, node, pkg):
+        """
+        Uninstall the VPP packages
+
+        :param node: Node dictionary
+        :param pkg: The vpp packages
+        :type node: dict
+        :type pkg: string
+        """
+        cmd = 'dpkg --purge {}'.format(pkg)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.format(
+                cmd, node['host'], stdout, stderr))
+
+    def _uninstall_vpp_pkg_centos(self, node, pkg):
+        """
+        Uninstall the VPP packages
+
+        :param node: Node dictionary
+        :param pkg: The vpp packages
+        :type node: dict
+        :type pkg: string
+        """
+        cmd = 'yum -y remove {}'.format(pkg)
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.format(
+                cmd, node['host'], stdout, stderr))
+
+    def _uninstall_vpp_ubuntu(self, node):
+        """
+        Uninstall the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :type node: dict
+        """
+        pkgs = self.get_installed_vpp_pkgs()
+
+        if len(pkgs) > 0:
+            if 'version' in pkgs[0]:
+                logging.info("Uninstall Ubuntu Packages")
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-python')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-java')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-api-lua')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-plugins')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dpdk-dev')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dpdk-dkms')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dev')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-dbg')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp')
+                self._uninstall_vpp_pkg_ubuntu(node, 'vpp-lib')
+            else:
+                logging.info("Uninstall locally installed Ubuntu Packages")
+                for pkg in pkgs:
+                    self._uninstall_vpp_pkg_ubuntu(node, pkg['name'])
+        else:
+            logging.error("There are no Ubuntu packages installed")
+
+    def _uninstall_vpp_centos(self, node):
+        """
+        Uninstall the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :type node: dict
+            """
+
+        pkgs = self.get_installed_vpp_pkgs()
+
+        if len(pkgs) > 0:
+            if 'version' in pkgs[0]:
+                logging.info("Uninstall CentOS Packages")
+                self._uninstall_vpp_pkg_centos(node, 'vpp-api-python')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-api-java')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-api-lua')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-plugins')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-dpdk-devel')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-devel')
+                self._uninstall_vpp_pkg_centos(node, 'vpp')
+                self._uninstall_vpp_pkg_centos(node, 'vpp-lib')
+            else:
+                logging.info("Uninstall locally installed CentOS Packages")
+                for pkg in pkgs:
+                    self._uninstall_vpp_pkg_centos(node, pkg['name'])
+        else:
+            logging.error("There are no CentOS packages installed")
+
+    def uninstall_vpp(self, node):
+        """
+        Uninstall the VPP packages
+
+        :param node: Node dictionary with cpuinfo.
+        :type node: dict
+        """
+        distro = self.get_linux_distro()
+        if distro[0] == 'Ubuntu':
+            self._uninstall_vpp_ubuntu(node)
+        elif distro[0] == 'CentOS Linux':
+            logging.info("Uninstall CentOS")
+            self._uninstall_vpp_centos(node)
+        else:
+            return
+
+    def show_vpp_settings(self, *additional_cmds):
+        """
+        Print default VPP settings. In case others are needed, can be
+        accepted as next parameters (each setting one parameter), preferably
+        in form of a string.
+
+        :param additional_cmds: Additional commands that the vpp should print
+        settings for.
+        :type additional_cmds: tuple
+        """
+        def_setting_tb_displayed = {
+            'IPv6 FIB': 'ip6 fib',
+            'IPv4 FIB': 'ip fib',
+            'Interface IP': 'int addr',
+            'Interfaces': 'int',
+            'ARP': 'ip arp',
+            'Errors': 'err'
+        }
+
+        if additional_cmds:
+            for cmd in additional_cmds:
+                def_setting_tb_displayed['Custom Setting: {}'.format(cmd)] \
+                    = cmd
+
+                for _, value in def_setting_tb_displayed.items():
+                    self.exec_command('vppctl sh {}'.format(value))
+
+    @staticmethod
+    def get_hardware(node):
+        """
+        Get the VPP hardware information and return it in a
+        dictionary
+
+        :param node: VPP node.
+        :type node: dict
+        :returns: Dictionary containing improtant VPP information
+        :rtype: dictionary
+        """
+
+        interfaces = {}
+        cmd = 'vppctl show hard'
+        (ret, stdout, stderr) = VPPUtil.exec_command(cmd)
+        if ret != 0:
+            return interfaces
+
+        lines = stdout.split('\n')
+        if len(lines[0]) is not 0:
+            if lines[0].split(' ')[0] == 'FileNotFoundError':
+                return interfaces
+
+        for line in lines:
+            if len(line) is 0:
+                continue
+
+            # If the first character is not whitespace
+            # create a new interface
+            if len(re.findall(r'\s', line[0])) is 0:
+                spl = line.split()
+                name = spl[0]
+                interfaces[name] = {}
+                interfaces[name]['index'] = spl[1]
+                interfaces[name]['state'] = spl[2]
+
+            # Ethernet address
+            rfall = re.findall(r'Ethernet address', line)
+            if rfall:
+                spl = line.split()
+                interfaces[name]['mac'] = spl[2]
+
+            # Carrier
+            rfall = re.findall(r'carrier', line)
+            if rfall:
+                spl = line.split('carrier ')
+                interfaces[name]['carrier'] = spl[1]
+
+            # Socket
+            rfall = re.findall(r'cpu socket', line)
+            if rfall:
+                spl = line.split('cpu socket ')
+                interfaces[name]['cpu socket'] = spl[1]
+
+            # Queues and Descriptors
+            rfall = re.findall(r'rx queues', line)
+            if rfall:
+                spl = line.split(',')
+                interfaces[name]['rx queues'] = spl[0].lstrip(' ').split(' ')[2]
+                interfaces[name]['rx descs'] = spl[1].split(' ')[3]
+                interfaces[name]['tx queues'] = spl[2].split(' ')[3]
+                interfaces[name]['tx descs'] = spl[3].split(' ')[3]
+
+        return interfaces
+
+    def _get_installed_vpp_pkgs_ubuntu(self):
+        """
+        Get the VPP hardware information and return it in a
+        dictionary
+
+        :returns: List of the packages installed
+        :rtype: list
+        """
+
+        pkgs = []
+        cmd = 'dpkg -l | grep vpp'
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            return pkgs
+
+        lines = stdout.split('\n')
+        for line in lines:
+            items = line.split()
+            if len(items) < 2:
+                continue
+            pkg = {'name': items[1], 'version': items[2]}
+            pkgs.append(pkg)
+
+        return pkgs
+
+    def _get_installed_vpp_pkgs_centos(self):
+        """
+        Get the VPP hardware information and return it in a
+        dictionary
+
+        :returns: List of the packages installed
+        :rtype: list
+        """
+
+        pkgs = []
+        cmd = 'rpm -qa | grep vpp'
+        (ret, stdout, stderr) = self.exec_command(cmd)
+        if ret != 0:
+            return pkgs
+
+        lines = stdout.split('\n')
+        for line in lines:
+            if len(line) == 0:
+                continue
+
+            items = line.split()
+            if len(items) < 2:
+                pkg = {'name': items[0]}
+            else:
+                pkg = {'name': items[1], 'version': items[2]}
+
+            pkgs.append(pkg)
+
+        return pkgs
+
+    def get_installed_vpp_pkgs(self):
+        """
+        Get the VPP hardware information and return it in a
+        dictionary
+
+        :returns: List of the packages installed
+        :rtype: list
+        """
+
+        distro = self.get_linux_distro()
+        if distro[0] == 'Ubuntu':
+            pkgs = self._get_installed_vpp_pkgs_ubuntu()
+        elif distro[0] == 'CentOS Linux':
+            pkgs = self._get_installed_vpp_pkgs_centos()
+        else:
+            return []
+
+        return pkgs
+
+    @staticmethod
+    def get_interfaces_numa_node(node, *iface_keys):
+        """Get numa node on which are located most of the interfaces.
+
+        Return numa node with highest count of interfaces provided as arguments.
+        Return 0 if the interface does not have numa_node information available.
+        If all interfaces have unknown location (-1), then return 0.
+        If most of interfaces have unknown location (-1), but there are
+        some interfaces with known location, then return the second most
+        location of the provided interfaces.
+
+        :param node: Node from DICT__nodes.
+        :param iface_keys: Interface keys for lookup.
+        :type node: dict
+        :type iface_keys: strings
+        """
+        numa_list = []
+        for if_key in iface_keys:
+            try:
+                numa_list.append(node['interfaces'][if_key].get('numa_node'))
+            except KeyError:
+                pass
+
+        numa_cnt_mc = Counter(numa_list).most_common()
+        numa_cnt_mc_len = len(numa_cnt_mc)
+        if numa_cnt_mc_len > 0 and numa_cnt_mc[0][0] != -1:
+            return numa_cnt_mc[0][0]
+        elif numa_cnt_mc_len > 1 and numa_cnt_mc[0][0] == -1:
+            return numa_cnt_mc[1][0]
+
+        return 0
+
+    @staticmethod
+    def start(node):
+        """
+
+        Starts vpp for a given node
+
+        :param node: VPP node.
+        :type node: dict
+        """
+
+        cmd = 'service vpp start'
+        (ret, stdout, stderr) = VPPUtil.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.
+                               format(cmd, node['host'],
+                                      stdout, stderr))
+
+    @staticmethod
+    def stop(node):
+        """
+
+        Stops vpp for a given node
+
+        :param node: VPP node.
+        :type node: dict
+        """
+
+        cmd = 'service vpp stop'
+        (ret, stdout, stderr) = VPPUtil.exec_command(cmd)
+        if ret != 0:
+            raise RuntimeError('{} failed on node {} {} {}'.
+                               format(cmd, node['host'],
+                                      stdout, stderr))
+
+    @staticmethod
+    def status(node):
+        """
+
+        Gets VPP status
+
+        :param: node
+        :type node: dict
+        :returns: status, errors
+        :rtype: tuple(str, list)
+        """
+        errors = []
+        vutil = VPPUtil()
+        pkgs = vutil.get_installed_vpp_pkgs()
+        if len(pkgs) == 0:
+            return "Not Installed", errors
+
+        cmd = 'service vpp status'
+        (ret, stdout, stderr) = VPPUtil.exec_command(cmd)
+
+        # Get the active status
+        state = re.findall(r'Active:[\w (\)]+', stdout)[0].split(' ')
+        if len(state) > 2:
+            statestr = "{} {}".format(state[1], state[2])
+        else:
+            statestr = "Invalid"
+
+        # For now we won't look for DPDK errors
+        # lines = stdout.split('\n')
+        # for line in lines:
+        #    if 'EAL' in line or \
+        #                     'FAILURE' in line or \
+        #                     'failed' in line or \
+        #                     'Failed' in line:
+        #         errors.append(line.lstrip(' '))
+
+        return statestr, errors
+
+    @staticmethod
+    def get_linux_distro():
+        """
+        Get the linux distribution and check if it is supported
+
+        :returns: linux distro, None if the distro is not supported
+        :rtype: list
+        """
+
+        distro = platform.linux_distribution()
+        if distro[0] == 'Ubuntu' or \
+                        distro[0] == 'CentOS Linux' or \
+                        distro[:26] == 'Linux Distribution Red Hat':
+            return distro
+        else:
+            raise RuntimeError('Linux Distribution {} is not supported'.format(distro[0]))
+
+    @staticmethod
+    def version():
+        """
+
+        Gets VPP Version information
+
+        :returns: version
+        :rtype: dict
+        """
+
+        version = {}
+        cmd = 'vppctl show version verbose'
+        (ret, stdout, stderr) = VPPUtil.exec_command(cmd)
+        if ret != 0:
+            return version
+
+        lines = stdout.split('\n')
+        if len(lines[0]) is not 0:
+            if lines[0].split(' ')[0] == 'FileNotFoundError':
+                return version
+
+        for line in lines:
+            if len(line) is 0:
+                continue
+            dct = line.split(':')
+            version[dct[0]] = dct[1].lstrip(' ')
+
+        return version