CSIT-870 Kubernetes/Ligato integration

- Update overall Kubernetes/Ligato integration
- CSIT-871 Update infrastructure / images
- CSIT-872 Update of KubernetesUtils.py

Change-Id: I7833f56abd351c694d3d3e1c9c45e649aec892e1
Signed-off-by: Peter Mikus <pmikus@cisco.com>

6 files changed, 368 insertions, 33 deletions

diff --git a/resources/libraries/bash/config/config b/resources/libraries/bash/config/config
new file mode 100644
index 0000000000..2b71edcc49
--- /dev/null
+++ b/resources/libraries/bash/config/config
@@ -0,0 +1,5 @@
+QEMU_INSTALL_DIR=/opt
+QEMU_INSTALL_VERSION=qemu-2.5.0
+
+DPDK_INSTALL_DIR=/opt
+DPDK_INSTALL_VERSION=dpdk-17.11
\ No newline at end of file
diff --git a/resources/libraries/bash/config/defaults b/resources/libraries/bash/config/defaults
new file mode 100644
index 0000000000..1547452982
--- /dev/null
+++ b/resources/libraries/bash/config/defaults
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+typeset -A cfg
+cfg=( # set default values in config array
+    [QEMU_INSTALL_DIR]="/opt"
+    [QEMU_INSTALL_VERSION]="qemu-2.5.0"
+    [DPDK_INSTALL_DIR]=/opt
+    [DPDK_INSTALL_VERSION]=dpdk-17.11
+    [K8S_CALICO]="${SCRIPT_DIR}/../../templates/kubernetes/calico_v2.6.3.yaml"
+    [K8S_CONTIV_VPP]="https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml"
+    [K8S_CSIT]="${SCRIPT_DIR}/../../templates/kubernetes/csit.yaml"
+)
\ No newline at end of file
diff --git a/resources/libraries/bash/k8s_setup.sh b/resources/libraries/bash/k8s_setup.sh
index 0649c711c6..f9f6c61ec7 100755
--- a/resources/libraries/bash/k8s_setup.sh
+++ b/resources/libraries/bash/k8s_setup.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -16,39 +16,59 @@ set -xo pipefail
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-K8S_CALICO="${SCRIPT_DIR}/../../templates/kubernetes/calico_v2.4.1.yaml"
-K8S_CSIT="${SCRIPT_DIR}/../../templates/kubernetes/csit.yaml"
+# Include
+source ${SCRIPT_DIR}/config/defaults
+source ${SCRIPT_DIR}/shell/dpdk_utils.sh
+source ${SCRIPT_DIR}/shell/k8s_utils.sh
 
-trap "sudo kubeadm reset && sudo rm -rf $HOME/.kube" ERR
+# Read configuration
+while read line
+do
+    if echo $line | grep -F = &>/dev/null
+    then
+        varname=$(echo "$line" | cut -d '=' -f 1)
+        cfg[$varname]=$(echo "$line" | cut -d '=' -f 2-)
+    fi
+done < ${script_dir}/../config/config
 
-# Revert any changes made to this host by 'kubeadm init' or 'kubeadm join'
-sudo kubeadm reset && sudo rm -rf $HOME/.kube || \
-    { echo "Failed to reset kubeadm"; exit 1; }
+trap "k8s_utils.destroy" ERR
 
-# Ret up the Kubernetes master
-sudo -E kubeadm init --token-ttl 0 --pod-network-cidr=192.168.0.0/16 || \
-    { echo "Failed to init kubeadm"; exit 1; }
+case "$1" in
+    prepare)
+        # Revert any changes made to this host by 'kubeadm init'
+        k8s_utils.destroy
+        # Sets up the Kubernetes master
+        k8s_utils.prepare
+        ;;
+    deploy_calico)
+        # Revert any changes made to this host by 'kubeadm init'
+        k8s_utils.destroy
+        # Load kernel modules uio/uio_pci_generic
+        dpdk_utils.load_modules
+        # Sets up the Kubernetes master
+        k8s_utils.prepare "--pod-network-cidr=192.168.0.0/16"
+        # Apply resources
+        k8s_utils.calico_deploy ${cfg[K8S_CALICO]}
+        # Dump Kubernetes objects ...
+        k8s_utils.dump_all
+        ;;
+    affinity_non_vpp)
+        # Set affinity for all non VPP docker containers to CPU 0
+        k8s_utils.affinity_non_vpp
+        ;;
+    destroy)
+        # Revert any changes made to this host by 'kubeadm init'
+        k8s_utils.destroy
+        ;;
+    *)
+        echo "usage: $0 function"
+        echo "function:"
+        echo "    prepare"
+        echo "    deploy_calico"
+        echo "    affinity_non_vpp"
+        echo "    destroy"
+        exit 1
+esac
+shift
 
-# Make cgroup non-exclusive for CPU and MEM
-sudo cgset -r cpuset.cpu_exclusive=0 /kubepods
-sudo cgset -r cpuset.mem_exclusive=0 /kubepods
-
-rm -rf $HOME/.kube
-mkdir -p $HOME/.kube
-sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-
-# Apply resources
-kubectl apply -f ${K8S_CALICO}  || \
-    { echo "Failed to apply Calico resources"; exit 1; }
-kubectl apply -f ${K8S_CSIT}  || \
-    { echo "Failed to apply CSIT resource"; exit 1; }
-
-# Update the taints
-kubectl taint nodes --all node-role.kubernetes.io/master- || \
-    { echo "Failed to taint nodes"; exit 1; }
-
-# Dump Kubernetes objects ...
-kubectl get all --all-namespaces
-
-echo Kubernetes is ready
+echo Kubernetes setup finished
diff --git a/resources/libraries/bash/shell/dpdk_utils.sh b/resources/libraries/bash/shell/dpdk_utils.sh
new file mode 100644
index 0000000000..9f81aaabce
--- /dev/null
+++ b/resources/libraries/bash/shell/dpdk_utils.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function dpdk_utils.dpdk_delete {
+    # Deletes the DPDK directory
+    # DPDK install directory
+    dpdk_install_dir=$1
+    # DPDK install version
+    dpdk_install_ver=$2
+
+    [ -d ${dpdk_install_dir}/${dpdk_install_ver} ] && \
+        sudo rm -r ${dpdk_install_dir}/${dpdk_install_ver} && \
+        echo "${dpdk_install_dir}/${dpdk_install_ver} removed"
+}
+
+function dpdk_utils.dpdk_install {
+    # Downloads and installs DPDK
+    # DPDK install directory
+    dpdk_install_dir=$1
+    # DPDK install version
+    dpdk_install_ver=$2
+    # DPDK compile target
+    dpdk_target=x86_64-native-linuxapp-gcc
+    # Force install (if true then remove previous installation; default false)
+    force_install=${3:-false}
+
+    if [ "$force_install" = true ]; then
+        # Cleanup DPDK DIR
+        dpdk_utils.dpdk_delete ${dpdk_install_dir} ${dpdk_install_ver}
+    else
+        # Test if DPDK was installed previously
+        test -d ${dpdk_install_dir}/${dpdk_install_ver} && \
+            { echo "DPDK ${dpdk_install_ver} ready"; exit 0; }
+    fi
+
+    # Download the DPDK package if no local copy exists
+    if [ ! -f ${dpdk_install_dir}/${dpdk_install_ver}.tar.xz ]; then
+        sudo wget -e use_proxy=yes -P ${dpdk_install_dir} -q \
+            fast.dpdk.org/rel/${dpdk_install_ver}.tar.xz || \
+            { echo "Failed to download ${dpdk_install_ver}"; exit 1; }
+    fi
+
+    # Create DPDK install dir if not exists and extract
+    sudo mkdir -p ${dpdk_install_dir} || \
+        { echo "Failed to create ${dpdk_install_dir}"; exit 1; }
+    sudo tar -xJf ${dpdk_install_dir}/${dpdk_install_ver}.tar.xz \
+        -C ${dpdk_install_dir} || \
+        { echo "Failed to extract ${dpdk_install_ver}.tar.xz"; exit 1; }
+
+    cd ${dpdk_install_dir}/${dpdk_install_ver}
+
+    # Compile and install the DPDK
+    sudo make install T=${dpdk_target} -j DESTDIR=install || \
+        { echo "Installation of ${dpdk_install_ver} failed"; exit 1; }
+
+    echo "DPDK ${dpdk_install_ver} ready"
+}
+
+function dpdk_utils.load_modules {
+    # Loads kernel modules and bind interfaces to drivers
+    # Use igb_uio [true|false]
+    use_igb_uio=${1:-false}
+    # DPDK install directory
+    dpdk_install_dir=$2
+    # DPDK install version
+    dpdk_install_ver=$3
+
+    sudo modprobe uio
+    sudo modprobe uio_pci_generic
+
+    if [ "${use_igb_uio}" = true ]; then
+        sudo rmmod igb_uio
+        # Try to insert IGB_UIO module
+        sudo insmod ${dpdk_install_dir}/${dpdk_install_ver}/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+        # If failed then download/compile DPDK
+        if [ $? -ne 0 ]; then
+            dpdk_utils.dpdk_install ${dpdk_install_dir} ${dpdk_install_ver} true
+            sudo insmod ${dpdk_install_dir}/${dpdk_install_ver}/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+        fi
+    fi
+}
diff --git a/resources/libraries/bash/shell/k8s_utils.sh b/resources/libraries/bash/shell/k8s_utils.sh
new file mode 100644
index 0000000000..55d3d815c9
--- /dev/null
+++ b/resources/libraries/bash/shell/k8s_utils.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function k8s_utils.destroy {
+    # Destroy existing Kubernetes deployment
+    kubectl drain $HOSTNAME --delete-local-data --force --ignore-daemonsets
+    kubectl delete node $HOSTNAME
+
+    # Revert any changes made to this host by 'kubeadm init' or 'kubeadm join'
+    sudo kubeadm reset && sudo rm -rf $HOME/.kube || \
+        { echo "Failed to reset kubeadm"; exit 1; }
+}
+
+function k8s_utils.prepare {
+    # Sets up the Kubernetes master
+
+    # Disable swap
+    sudo swapoff --all
+
+    # Set up the Kubernetes master
+    sudo -E kubeadm init --token-ttl 0 ${1} || \
+        { echo "Failed to init kubeadm"; exit 1; }
+
+    # Make cgroup non-exclusive for CPU and MEM
+    sudo cgset -r cpuset.cpu_exclusive=0 /kubepods
+    sudo cgset -r cpuset.mem_exclusive=0 /kubepods
+
+    rm -rf $HOME/.kube
+    mkdir -p $HOME/.kube
+    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+    sudo chown $(id -u):$(id -g) $HOME/.kube/config
+}
+
+function k8s_utils.taint {
+    # Updates the taints
+    kubectl taint nodes --all node-role.kubernetes.io/master- || \
+        { echo "Failed to taint nodes"; exit 1; }
+}
+
+function k8s_utils.calico_deploy {
+    # Calico yaml URL or file
+    k8s_calico=$1
+
+    # Apply resources
+    kubectl apply -f ${k8s_calico}  || \
+        { echo "Failed to apply ${k8s_calico}"; exit 1; }
+
+    # Update the taints
+    k8s_utils.taint
+}
+
+function k8s_utils.contiv_vpp_deploy {
+    # Contiv yaml URL or file
+    k8s_contiv=$1
+    k8s_contiv_patch="kubecon.contiv-vpp-yaml-patch.diff"
+
+    # Pull the most recent Docker images
+    bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/pull-images.sh)
+
+    # Apply resources
+    wget ${k8s_contiv}
+    patch contiv-vpp.yaml -i ${k8s_contiv_patch} -o - | kubectl apply -f - || \
+        { echo "Failed to apply Contiv resources"; exit 1; }
+    rm contiv-vpp.yaml
+
+    # Update the taints
+    k8s_utils.taint
+}
+
+function k8s_utils.cri_shim_install {
+    # Install the CRI Shim on host
+    sudo su root -c 'bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/cri-install.sh)'
+}
+
+function k8s_utils.cri_shim_uninstall {
+    # Uninstall the CRI Shim on host
+    sudo su root -c 'bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/cri-install.sh) --uninstall'
+}
+
+function k8s_utils.kube_proxy_install {
+    # Installing custom version of Kube-Proxy to enable Kubernetes services
+    bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/proxy-install.sh)
+}
+
+function k8s_utils.apply {
+    # Resource yaml URL or file
+    k8s_resource=$1
+
+    # Apply resources
+    kubectl apply -f ${k8s_resource}  || \
+        { echo "Failed to apply ${k8s_resource}"; exit 1; }
+}
+
+function k8s_utils.resource_delete {
+    # Resource yaml URL or file
+    k8s_resource=$1
+
+    # Delete resources
+    kubectl delete -f ${k8s_resource}  || \
+        { echo "Failed to delete ${k8s_resource}"; exit 1; }
+}
+
+function k8s_utils.affinity_non_vpp {
+    # Set affinity for all non VPP docker containers to CPU 0
+    for i in `sudo docker ps --format "{{.ID}} {{.Names}}" | grep -v vpp | cut -d' ' -f1`; do
+        sudo docker update --cpuset-cpus 0 ${i}
+    done
+}
+
+function k8s_utils.dump_all {
+    # Dumps the kubernetes objects
+    kubectl get all --all-namespaces
+    kubectl describe nodes
+}
diff --git a/resources/libraries/bash/shell/qemu_utils.sh b/resources/libraries/bash/shell/qemu_utils.sh
new file mode 100644
index 0000000000..510d9f2838
--- /dev/null
+++ b/resources/libraries/bash/shell/qemu_utils.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function qemu_utils.qemu_delete {
+    # Deletes the QEMU directory
+    # QEMU install directory
+    qemu_install_dir=$1
+    # QEMU install version
+    qemu_install_ver=$2
+
+    [ -d ${qemu_install_dir}/${qemu_install_ver} ] && \
+        sudo rm -r ${qemu_install_dir}/${qemu_install_ver} && \
+        echo "${qemu_install_dir}/${qemu_install_ver} removed"
+}
+
+function qemu_utils.qemu_install {
+    # Downloads and installs QEMU
+    # QEMU install directory
+    qemu_install_dir=$1
+    # QEMU install version
+    qemu_install_ver=$2
+    # QEMU patch
+    qemu_patch=$3
+    # Force install (if true then remove previous installation; default false)
+    force_install=${4:-false}
+    # QEMU repo URL
+    qemu_package_url="http://download.qemu-project.org/${qemu_install_ver}.tar.xz"
+
+    if [ $force_install ]; then
+        # Cleanup QEMU dir
+        qemu_utils.qemu_delete $qemu_install_dir $qemu_install_ver
+    else
+        # Test if QEMU was installed previously
+        test -d $qemu_install_dir && \
+            { echo "Qemu already installed: $qemu_install_dir"; exit 0; }
+    fi
+
+    tmp_dir=$(mktemp -d) || \
+        { echo "Failed to create temporary working dir"; exit 1; }
+    trap "rm -r ${tmp_dir}" EXIT
+
+    # Download QEMU source code if no local copy exists
+    if [ ! -f /opt/${qemu_install_ver}.tar.xz ]; then
+        sudo wget -e use_proxy=yes -P /opt -q ${qemu_package_url} || \
+            { echo "Failed to download ${qemu_install_ver}"; exit 1; }
+    fi
+    tar --strip-components 1 -xvJf ${tmp_dir}/${qemu_install_ver}.tar.xz -C ${tmp_dir} && \
+        { echo "Failed to exctract ${qemu_install_ver}.tar.xz"; exit 1; }
+
+    cd ${tmp_dir}
+    sudo mkdir -p ${qemu_install_dir} || \
+        { echo "Failed to create ${qemu_install_dir}"; exit 1; }
+
+    # Apply additional patches
+    if [ $qemu_patch ]
+    then
+        chmod +x ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver}/*
+        run-parts --verbose --report  ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver}
+    fi
+
+    # Build
+    sudo ./configure --target-list=x86_64-softmmu --prefix=${qemu_install_dir}/${qemu_install_ver} || \
+        { echo "Failed to configure ${qemu_install_ver}"; exit 1; }
+    sudo make -j`nproc` || \
+        { echo "Failed to compile ${qemu_install_ver}"; exit 1; }
+    sudo make install || \
+        { echo "Failed to install ${qemu_install_ver}"; exit 1; }
+
+    echo "QEMU ${qemu_install_ver} ready"
+}
\ No newline at end of file