From 0437095f4bf958154f25a3f163f432b22fcdc743 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Tue, 5 Dec 2017 14:44:32 +0100 Subject: CSIT-870 Kubernetes/Ligato integration - Update overall Kubernetes/Ligato integration - CSIT-871 Update infrastructure / images - CSIT-872 Update of KubernetesUtils.py Change-Id: I7833f56abd351c694d3d3e1c9c45e649aec892e1 Signed-off-by: Peter Mikus --- bootstrap-verify-perf-ligato.sh | 9 +- resources/libraries/bash/config/config | 5 + resources/libraries/bash/config/defaults | 12 + resources/libraries/bash/k8s_setup.sh | 86 ++-- resources/libraries/bash/shell/dpdk_utils.sh | 92 +++++ resources/libraries/bash/shell/k8s_utils.sh | 125 ++++++ resources/libraries/bash/shell/qemu_utils.sh | 81 ++++ resources/libraries/python/KubernetesUtils.py | 306 ++++++++------ .../robot/performance/performance_setup.robot | 26 ++ resources/templates/kubernetes/calico_v2.4.1.yaml | 387 ----------------- resources/templates/kubernetes/calico_v2.6.3.yaml | 459 +++++++++++++++++++++ ...eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc.yaml | 72 ++++ ...eth-1drcl2bdbasemaclrn-eth-8memif-4drcl2xc.yaml | 108 +++++ .../ch-eth-1drcl2xcbase-eth-4memif-2drcl2xc.yaml | 72 ++++ .../ch-eth-1drcl2xcbase-eth-8memif-4drcl2xc.yaml | 108 +++++ ...eth-1drcl2bdbasemaclrn-eth-2memif-2drcl2xc.yaml | 72 ++++ ...eth-1drcl2bdbasemaclrn-eth-2memif-4drcl2xc.yaml | 108 +++++ .../ho-eth-1drcl2xcbase-eth-2memif-2drcl2xc.yaml | 72 ++++ .../ho-eth-1drcl2xcbase-eth-2memif-4drcl2xc.yaml | 108 +++++ ...eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc.yaml | 62 +++ .../pa-eth-1drcl2xcbase-eth-2memif-1drcl2xc.yaml | 54 +++ .../kubernetes/configmaps/vnf-agent-cfg.yaml | 15 + .../kubernetes/configmaps/vswitch-agent-cfg.yaml | 15 + resources/templates/kubernetes/csit.yaml | 4 - resources/templates/kubernetes/etcd.yaml | 25 -- ...eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc.yaml | 206 --------- ...eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc.yaml | 271 ------------ .../eth-1drcl2xcbase-eth-2memif-1drcl2xc.yaml | 206 --------- .../eth-1drcl2xcbase-eth-4memif-2drcl2xc.yaml | 271 ------------ resources/templates/kubernetes/kafka.yaml | 22 - .../templates/kubernetes/namespaces/csit.yaml | 4 + .../kubernetes/pods/contiv-sfc-controller.yaml | 22 + .../templates/kubernetes/pods/contiv-vnf.yaml | 45 ++ .../templates/kubernetes/pods/contiv-vswitch.yaml | 52 +++ resources/templates/kubernetes/pods/etcdv3.yaml | 25 ++ resources/templates/kubernetes/pods/kafka.yaml | 22 + tests/kubernetes/perf/__init__.robot | 5 +- ...maclrn-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot | 310 ++++++++++++++ ...maclrn-eth-8memif-4drcl2xc-k8s-ndrpdrdisc.robot | 324 +++++++++++++++ ...xcbase-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot | 309 ++++++++++++++ ...xcbase-eth-8memif-4drcl2xc-k8s-ndrpdrdisc.robot | 323 +++++++++++++++ ...maclrn-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot | 303 -------------- ...maclrn-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot | 306 -------------- ...xcbase-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot | 303 -------------- ...xcbase-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot | 306 -------------- ...maclrn-eth-2memif-2drcl2xc-k8s-ndrpdrdisc.robot | 310 ++++++++++++++ ...maclrn-eth-2memif-4drcl2xc-k8s-ndrpdrdisc.robot | 324 +++++++++++++++ ...xcbase-eth-2memif-2drcl2xc-k8s-ndrpdrdisc.robot | 309 ++++++++++++++ ...xcbase-eth-2memif-4drcl2xc-k8s-ndrpdrdisc.robot | 323 +++++++++++++++ ...maclrn-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot | 309 ++++++++++++++ ...xcbase-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot | 302 ++++++++++++++ 51 files changed, 5224 insertions(+), 2771 deletions(-) create mode 100644 resources/libraries/bash/config/config create mode 100644 resources/libraries/bash/config/defaults create mode 100644 resources/libraries/bash/shell/dpdk_utils.sh create mode 100644 resources/libraries/bash/shell/k8s_utils.sh create mode 100644 resources/libraries/bash/shell/qemu_utils.sh delete mode 100644 resources/templates/kubernetes/calico_v2.4.1.yaml create mode 100644 resources/templates/kubernetes/calico_v2.6.3.yaml create mode 100644 resources/templates/kubernetes/configmaps/ch-eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ch-eth-1drcl2bdbasemaclrn-eth-8memif-4drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ch-eth-1drcl2xcbase-eth-4memif-2drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ch-eth-1drcl2xcbase-eth-8memif-4drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ho-eth-1drcl2bdbasemaclrn-eth-2memif-2drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ho-eth-1drcl2bdbasemaclrn-eth-2memif-4drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ho-eth-1drcl2xcbase-eth-2memif-2drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/ho-eth-1drcl2xcbase-eth-2memif-4drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/pa-eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/pa-eth-1drcl2xcbase-eth-2memif-1drcl2xc.yaml create mode 100644 resources/templates/kubernetes/configmaps/vnf-agent-cfg.yaml create mode 100644 resources/templates/kubernetes/configmaps/vswitch-agent-cfg.yaml delete mode 100644 resources/templates/kubernetes/csit.yaml delete mode 100644 resources/templates/kubernetes/etcd.yaml delete mode 100644 resources/templates/kubernetes/eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc.yaml delete mode 100644 resources/templates/kubernetes/eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc.yaml delete mode 100644 resources/templates/kubernetes/eth-1drcl2xcbase-eth-2memif-1drcl2xc.yaml delete mode 100644 resources/templates/kubernetes/eth-1drcl2xcbase-eth-4memif-2drcl2xc.yaml delete mode 100644 resources/templates/kubernetes/kafka.yaml create mode 100644 resources/templates/kubernetes/namespaces/csit.yaml create mode 100644 resources/templates/kubernetes/pods/contiv-sfc-controller.yaml create mode 100644 resources/templates/kubernetes/pods/contiv-vnf.yaml create mode 100644 resources/templates/kubernetes/pods/contiv-vswitch.yaml create mode 100644 resources/templates/kubernetes/pods/etcdv3.yaml create mode 100644 resources/templates/kubernetes/pods/kafka.yaml create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ch-eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ch-eth-1drcl2bdbasemaclrn-eth-8memif-4drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ch-eth-1drcl2xcbase-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ch-eth-1drcl2xcbase-eth-8memif-4drcl2xc-k8s-ndrpdrdisc.robot delete mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot delete mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-eth-1drcl2bdbasemaclrn-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot delete mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-eth-1drcl2xcbase-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot delete mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-eth-1drcl2xcbase-eth-4memif-2drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ho-eth-1drcl2bdbasemaclrn-eth-2memif-2drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ho-eth-1drcl2bdbasemaclrn-eth-2memif-4drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ho-eth-1drcl2xcbase-eth-2memif-2drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-ho-eth-1drcl2xcbase-eth-2memif-4drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-pa-eth-1drcl2bdbasemaclrn-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot create mode 100644 tests/kubernetes/perf/container_memif/10ge2p1x520-pa-eth-1drcl2xcbase-eth-2memif-1drcl2xc-k8s-ndrpdrdisc.robot diff --git a/bootstrap-verify-perf-ligato.sh b/bootstrap-verify-perf-ligato.sh index 10470dc9ea..e11b5e77b3 100644 --- a/bootstrap-verify-perf-ligato.sh +++ b/bootstrap-verify-perf-ligato.sh @@ -88,7 +88,7 @@ tar -zcvf ${SCRIPT_DIR}/vpp.tar.gz vpp/* && rm -R vpp LIGATO_REPO_URL=$(cat ${SCRIPT_DIR}/LIGATO_REPO_URL) VPP_AGENT_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_AGENT_STABLE_VER) VPP_AGENT_STABLE_COMMIT="$( expr match `cat VPP_AGENT_STABLE_VER` '.*g\(.*\)' )" -DOCKER_DEB="docker-ce_17.06.2~ce-0~ubuntu_amd64.deb" +DOCKER_DEB="docker-ce_17.09.0~ce-0~ubuntu_amd64.deb" # Clone & checkout stable vnf-agent cd .. && git clone ${LIGATO_REPO_URL}/vpp-agent @@ -123,16 +123,15 @@ sudo docker images cd ${SCRIPT_DIR}/../vpp-agent/docker/prod_vpp_agent/ &&\ mv ${SCRIPT_DIR}/vpp.tar.gz . &&\ ./extract_agent_files.sh &&\ - sudo docker build -t prod_vpp_agent --no-cache . &&\ - ./shrink.sh + sudo docker build -t prod_vpp_agent --no-cache . # Export Docker image -sudo docker save prod_vpp_agent_shrink | gzip > prod_vpp_agent_shrink.tar.gz +sudo docker save prod_vpp_agent | gzip > prod_vpp_agent.tar.gz # If image build fails, complain clearly and exit if [ $? != 0 ]; then echo "Failed to build vpp-agent Docker image." exit 1 fi -DOCKER_IMAGE="$( readlink -f prod_vpp_agent_shrink.tar.gz | tr '\n' ' ' )" +DOCKER_IMAGE="$( readlink -f prod_vpp_agent.tar.gz | tr '\n' ' ' )" cd ${SCRIPT_DIR} diff --git a/resources/libraries/bash/config/config b/resources/libraries/bash/config/config new file mode 100644 index 0000000000..2b71edcc49 --- /dev/null +++ b/resources/libraries/bash/config/config @@ -0,0 +1,5 @@ +QEMU_INSTALL_DIR=/opt +QEMU_INSTALL_VERSION=qemu-2.5.0 + +DPDK_INSTALL_DIR=/opt +DPDK_INSTALL_VERSION=dpdk-17.11 \ No newline at end of file diff --git a/resources/libraries/bash/config/defaults b/resources/libraries/bash/config/defaults new file mode 100644 index 0000000000..1547452982 --- /dev/null +++ b/resources/libraries/bash/config/defaults @@ -0,0 +1,12 @@ +#!/bin/bash + +typeset -A cfg +cfg=( # set default values in config array + [QEMU_INSTALL_DIR]="/opt" + [QEMU_INSTALL_VERSION]="qemu-2.5.0" + [DPDK_INSTALL_DIR]=/opt + [DPDK_INSTALL_VERSION]=dpdk-17.11 + [K8S_CALICO]="${SCRIPT_DIR}/../../templates/kubernetes/calico_v2.6.3.yaml" + [K8S_CONTIV_VPP]="https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml" + [K8S_CSIT]="${SCRIPT_DIR}/../../templates/kubernetes/csit.yaml" +) \ No newline at end of file diff --git a/resources/libraries/bash/k8s_setup.sh b/resources/libraries/bash/k8s_setup.sh index 0649c711c6..f9f6c61ec7 100755 --- a/resources/libraries/bash/k8s_setup.sh +++ b/resources/libraries/bash/k8s_setup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -16,39 +16,59 @@ set -xo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -K8S_CALICO="${SCRIPT_DIR}/../../templates/kubernetes/calico_v2.4.1.yaml" -K8S_CSIT="${SCRIPT_DIR}/../../templates/kubernetes/csit.yaml" +# Include +source ${SCRIPT_DIR}/config/defaults +source ${SCRIPT_DIR}/shell/dpdk_utils.sh +source ${SCRIPT_DIR}/shell/k8s_utils.sh -trap "sudo kubeadm reset && sudo rm -rf $HOME/.kube" ERR +# Read configuration +while read line +do + if echo $line | grep -F = &>/dev/null + then + varname=$(echo "$line" | cut -d '=' -f 1) + cfg[$varname]=$(echo "$line" | cut -d '=' -f 2-) + fi +done < ${script_dir}/../config/config -# Revert any changes made to this host by 'kubeadm init' or 'kubeadm join' -sudo kubeadm reset && sudo rm -rf $HOME/.kube || \ - { echo "Failed to reset kubeadm"; exit 1; } +trap "k8s_utils.destroy" ERR -# Ret up the Kubernetes master -sudo -E kubeadm init --token-ttl 0 --pod-network-cidr=192.168.0.0/16 || \ - { echo "Failed to init kubeadm"; exit 1; } +case "$1" in + prepare) + # Revert any changes made to this host by 'kubeadm init' + k8s_utils.destroy + # Sets up the Kubernetes master + k8s_utils.prepare + ;; + deploy_calico) + # Revert any changes made to this host by 'kubeadm init' + k8s_utils.destroy + # Load kernel modules uio/uio_pci_generic + dpdk_utils.load_modules + # Sets up the Kubernetes master + k8s_utils.prepare "--pod-network-cidr=192.168.0.0/16" + # Apply resources + k8s_utils.calico_deploy ${cfg[K8S_CALICO]} + # Dump Kubernetes objects ... + k8s_utils.dump_all + ;; + affinity_non_vpp) + # Set affinity for all non VPP docker containers to CPU 0 + k8s_utils.affinity_non_vpp + ;; + destroy) + # Revert any changes made to this host by 'kubeadm init' + k8s_utils.destroy + ;; + *) + echo "usage: $0 function" + echo "function:" + echo " prepare" + echo " deploy_calico" + echo " affinity_non_vpp" + echo " destroy" + exit 1 +esac +shift -# Make cgroup non-exclusive for CPU and MEM -sudo cgset -r cpuset.cpu_exclusive=0 /kubepods -sudo cgset -r cpuset.mem_exclusive=0 /kubepods - -rm -rf $HOME/.kube -mkdir -p $HOME/.kube -sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config -sudo chown $(id -u):$(id -g) $HOME/.kube/config - -# Apply resources -kubectl apply -f ${K8S_CALICO} || \ - { echo "Failed to apply Calico resources"; exit 1; } -kubectl apply -f ${K8S_CSIT} || \ - { echo "Failed to apply CSIT resource"; exit 1; } - -# Update the taints -kubectl taint nodes --all node-role.kubernetes.io/master- || \ - { echo "Failed to taint nodes"; exit 1; } - -# Dump Kubernetes objects ... -kubectl get all --all-namespaces - -echo Kubernetes is ready +echo Kubernetes setup finished diff --git a/resources/libraries/bash/shell/dpdk_utils.sh b/resources/libraries/bash/shell/dpdk_utils.sh new file mode 100644 index 0000000000..9f81aaabce --- /dev/null +++ b/resources/libraries/bash/shell/dpdk_utils.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function dpdk_utils.dpdk_delete { + # Deletes the DPDK directory + # DPDK install directory + dpdk_install_dir=$1 + # DPDK install version + dpdk_install_ver=$2 + + [ -d ${dpdk_install_dir}/${dpdk_install_ver} ] && \ + sudo rm -r ${dpdk_install_dir}/${dpdk_install_ver} && \ + echo "${dpdk_install_dir}/${dpdk_install_ver} removed" +} + +function dpdk_utils.dpdk_install { + # Downloads and installs DPDK + # DPDK install directory + dpdk_install_dir=$1 + # DPDK install version + dpdk_install_ver=$2 + # DPDK compile target + dpdk_target=x86_64-native-linuxapp-gcc + # Force install (if true then remove previous installation; default false) + force_install=${3:-false} + + if [ "$force_install" = true ]; then + # Cleanup DPDK DIR + dpdk_utils.dpdk_delete ${dpdk_install_dir} ${dpdk_install_ver} + else + # Test if DPDK was installed previously + test -d ${dpdk_install_dir}/${dpdk_install_ver} && \ + { echo "DPDK ${dpdk_install_ver} ready"; exit 0; } + fi + + # Download the DPDK package if no local copy exists + if [ ! -f ${dpdk_install_dir}/${dpdk_install_ver}.tar.xz ]; then + sudo wget -e use_proxy=yes -P ${dpdk_install_dir} -q \ + fast.dpdk.org/rel/${dpdk_install_ver}.tar.xz || \ + { echo "Failed to download ${dpdk_install_ver}"; exit 1; } + fi + + # Create DPDK install dir if not exists and extract + sudo mkdir -p ${dpdk_install_dir} || \ + { echo "Failed to create ${dpdk_install_dir}"; exit 1; } + sudo tar -xJf ${dpdk_install_dir}/${dpdk_install_ver}.tar.xz \ + -C ${dpdk_install_dir} || \ + { echo "Failed to extract ${dpdk_install_ver}.tar.xz"; exit 1; } + + cd ${dpdk_install_dir}/${dpdk_install_ver} + + # Compile and install the DPDK + sudo make install T=${dpdk_target} -j DESTDIR=install || \ + { echo "Installation of ${dpdk_install_ver} failed"; exit 1; } + + echo "DPDK ${dpdk_install_ver} ready" +} + +function dpdk_utils.load_modules { + # Loads kernel modules and bind interfaces to drivers + # Use igb_uio [true|false] + use_igb_uio=${1:-false} + # DPDK install directory + dpdk_install_dir=$2 + # DPDK install version + dpdk_install_ver=$3 + + sudo modprobe uio + sudo modprobe uio_pci_generic + + if [ "${use_igb_uio}" = true ]; then + sudo rmmod igb_uio + # Try to insert IGB_UIO module + sudo insmod ${dpdk_install_dir}/${dpdk_install_ver}/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + # If failed then download/compile DPDK + if [ $? -ne 0 ]; then + dpdk_utils.dpdk_install ${dpdk_install_dir} ${dpdk_install_ver} true + sudo insmod ${dpdk_install_dir}/${dpdk_install_ver}/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + fi + fi +} diff --git a/resources/libraries/bash/shell/k8s_utils.sh b/resources/libraries/bash/shell/k8s_utils.sh new file mode 100644 index 0000000000..55d3d815c9 --- /dev/null +++ b/resources/libraries/bash/shell/k8s_utils.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function k8s_utils.destroy { + # Destroy existing Kubernetes deployment + kubectl drain $HOSTNAME --delete-local-data --force --ignore-daemonsets + kubectl delete node $HOSTNAME + + # Revert any changes made to this host by 'kubeadm init' or 'kubeadm join' + sudo kubeadm reset && sudo rm -rf $HOME/.kube || \ + { echo "Failed to reset kubeadm"; exit 1; } +} + +function k8s_utils.prepare { + # Sets up the Kubernetes master + + # Disable swap + sudo swapoff --all + + # Set up the Kubernetes master + sudo -E kubeadm init --token-ttl 0 ${1} || \ + { echo "Failed to init kubeadm"; exit 1; } + + # Make cgroup non-exclusive for CPU and MEM + sudo cgset -r cpuset.cpu_exclusive=0 /kubepods + sudo cgset -r cpuset.mem_exclusive=0 /kubepods + + rm -rf $HOME/.kube + mkdir -p $HOME/.kube + sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config + sudo chown $(id -u):$(id -g) $HOME/.kube/config +} + +function k8s_utils.taint { + # Updates the taints + kubectl taint nodes --all node-role.kubernetes.io/master- || \ + { echo "Failed to taint nodes"; exit 1; } +} + +function k8s_utils.calico_deploy { + # Calico yaml URL or file + k8s_calico=$1 + + # Apply resources + kubectl apply -f ${k8s_calico} || \ + { echo "Failed to apply ${k8s_calico}"; exit 1; } + + # Update the taints + k8s_utils.taint +} + +function k8s_utils.contiv_vpp_deploy { + # Contiv yaml URL or file + k8s_contiv=$1 + k8s_contiv_patch="kubecon.contiv-vpp-yaml-patch.diff" + + # Pull the most recent Docker images + bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/pull-images.sh) + + # Apply resources + wget ${k8s_contiv} + patch contiv-vpp.yaml -i ${k8s_contiv_patch} -o - | kubectl apply -f - || \ + { echo "Failed to apply Contiv resources"; exit 1; } + rm contiv-vpp.yaml + + # Update the taints + k8s_utils.taint +} + +function k8s_utils.cri_shim_install { + # Install the CRI Shim on host + sudo su root -c 'bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/cri-install.sh)' +} + +function k8s_utils.cri_shim_uninstall { + # Uninstall the CRI Shim on host + sudo su root -c 'bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/cri-install.sh) --uninstall' +} + +function k8s_utils.kube_proxy_install { + # Installing custom version of Kube-Proxy to enable Kubernetes services + bash <(curl -s https://raw.githubusercontent.com/contiv/vpp/master/k8s/proxy-install.sh) +} + +function k8s_utils.apply { + # Resource yaml URL or file + k8s_resource=$1 + + # Apply resources + kubectl apply -f ${k8s_resource} || \ + { echo "Failed to apply ${k8s_resource}"; exit 1; } +} + +function k8s_utils.resource_delete { + # Resource yaml URL or file + k8s_resource=$1 + + # Delete resources + kubectl delete -f ${k8s_resource} || \ + { echo "Failed to delete ${k8s_resource}"; exit 1; } +} + +function k8s_utils.affinity_non_vpp { + # Set affinity for all non VPP docker containers to CPU 0 + for i in `sudo docker ps --format "{{.ID}} {{.Names}}" | grep -v vpp | cut -d' ' -f1`; do + sudo docker update --cpuset-cpus 0 ${i} + done +} + +function k8s_utils.dump_all { + # Dumps the kubernetes objects + kubectl get all --all-namespaces + kubectl describe nodes +} diff --git a/resources/libraries/bash/shell/qemu_utils.sh b/resources/libraries/bash/shell/qemu_utils.sh new file mode 100644 index 0000000000..510d9f2838 --- /dev/null +++ b/resources/libraries/bash/shell/qemu_utils.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function qemu_utils.qemu_delete { + # Deletes the QEMU directory + # QEMU install directory + qemu_install_dir=$1 + # QEMU install version + qemu_install_ver=$2 + + [ -d ${qemu_install_dir}/${qemu_install_ver} ] && \ + sudo rm -r ${qemu_install_dir}/${qemu_install_ver} && \ + echo "${qemu_install_dir}/${qemu_install_ver} removed" +} + +function qemu_utils.qemu_install { + # Downloads and installs QEMU + # QEMU install directory + qemu_install_dir=$1 + # QEMU install version + qemu_install_ver=$2 + # QEMU patch + qemu_patch=$3 + # Force install (if true then remove previous installation; default false) + force_install=${4:-false} + # QEMU repo URL + qemu_package_url="http://download.qemu-project.org/${qemu_install_ver}.tar.xz" + + if [ $force_install ]; then + # Cleanup QEMU dir + qemu_utils.qemu_delete $qemu_install_dir $qemu_install_ver + else + # Test if QEMU was installed previously + test -d $qemu_install_dir && \ + { echo "Qemu already installed: $qemu_install_dir"; exit 0; } + fi + + tmp_dir=$(mktemp -d) || \ + { echo "Failed to create temporary working dir"; exit 1; } + trap "rm -r ${tmp_dir}" EXIT + + # Download QEMU source code if no local copy exists + if [ ! -f /opt/${qemu_install_ver}.tar.xz ]; then + sudo wget -e use_proxy=yes -P /opt -q ${qemu_package_url} || \ + { echo "Failed to download ${qemu_install_ver}"; exit 1; } + fi + tar --strip-components 1 -xvJf ${tmp_dir}/${qemu_install_ver}.tar.xz -C ${tmp_dir} && \ + { echo "Failed to exctract ${qemu_install_ver}.tar.xz"; exit 1; } + + cd ${tmp_dir} + sudo mkdir -p ${qemu_install_dir} || \ + { echo "Failed to create ${qemu_install_dir}"; exit 1; } + + # Apply additional patches + if [ $qemu_patch ] + then + chmod +x ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver}/* + run-parts --verbose --report ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver} + fi + + # Build + sudo ./configure --target-list=x86_64-softmmu --prefix=${qemu_install_dir}/${qemu_install_ver} || \ + { echo "Failed to configure ${qemu_install_ver}"; exit 1; } + sudo make -j`nproc` || \ + { echo "Failed to compile ${qemu_install_ver}"; exit 1; } + sudo make install || \ + { echo "Failed to install ${qemu_install_ver}"; exit 1; } + + echo "QEMU ${qemu_install_ver} ready" +} \ No newline at end of file diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py index bcbb7f5dff..ceeab0b07e 100644 --- a/resources/libraries/python/KubernetesUtils.py +++ b/resources/libraries/python/KubernetesUtils.py @@ -13,8 +13,7 @@ """Library to control Kubernetes kubectl.""" -import time -import yaml +from time import sleep from resources.libraries.python.constants import Constants from resources.libraries.python.topology import NodeType @@ -24,6 +23,8 @@ from resources.libraries.python.VppConfigGenerator import VppConfigGenerator __all__ = ["KubernetesUtils"] +# Maximum number of retries to check if PODs are running or deleted. +MAX_RETRY = 48 class KubernetesUtils(object): """Kubernetes utilities class.""" @@ -43,9 +44,10 @@ class KubernetesUtils(object): ssh = SSH() ssh.connect(node) - cmd = '{dir}/{lib}/k8s_setup.sh '.format(dir=Constants.REMOTE_FW_DIR, - lib=Constants.RESOURCES_LIB_SH) - (ret_code, _, _) = ssh.exec_command(cmd, timeout=120) + cmd = '{dir}/{lib}/k8s_setup.sh deploy_calico'\ + .format(dir=Constants.REMOTE_FW_DIR, + lib=Constants.RESOURCES_LIB_SH) + (ret_code, _, _) = ssh.exec_command(cmd, timeout=240) if int(ret_code) != 0: raise RuntimeError('Failed to setup Kubernetes on {node}.' .format(node=node['host'])) @@ -64,6 +66,36 @@ class KubernetesUtils(object): if node['type'] == NodeType.DUT: KubernetesUtils.setup_kubernetes_on_node(node) + @staticmethod + def destroy_kubernetes_on_node(node): + """Destroy Kubernetes on node. + + :param node: DUT node. + :type node: dict + :raises RuntimeError: If destroying Kubernetes failed. + """ + ssh = SSH() + ssh.connect(node) + + cmd = '{dir}/{lib}/k8s_setup.sh destroy'\ + .format(dir=Constants.REMOTE_FW_DIR, + lib=Constants.RESOURCES_LIB_SH) + (ret_code, _, _) = ssh.exec_command(cmd, timeout=120) + if int(ret_code) != 0: + raise RuntimeError('Failed to destroy Kubernetes on {node}.' + .format(node=node['host'])) + + @staticmethod + def destroy_kubernetes_on_all_duts(nodes): + """Destroy Kubernetes on all DUTs. + + :param nodes: Topology nodes. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + KubernetesUtils.destroy_kubernetes_on_node(node) + @staticmethod def apply_kubernetes_resource_on_node(node, yaml_file, **kwargs): """Apply Kubernetes resource on node. @@ -79,18 +111,15 @@ class KubernetesUtils(object): ssh = SSH() ssh.connect(node) - stream = file('{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S, - yaml=yaml_file), 'r') - - for data in yaml.load_all(stream): + fqn_file = '{tpl}/{yaml}'.format(tpl=Constants.RESOURCES_TPL_K8S, + yaml=yaml_file) + with open(fqn_file, 'r') as src_file: + stream = src_file.read() data = reduce(lambda a, kv: a.replace(*kv), kwargs.iteritems(), - yaml.dump(data, default_flow_style=False)) - # Workaround to avoid using RAW string anotated with | in YAML as - # library + bash is misinterpreting spaces. - data = data.replace('.conf:\n', '.conf: |\n') + stream) cmd = 'cat <