Optimize Qemu installation to speed up vhost tests

Currently Qemu is being installed if there is a change of qsz parameter
between tests/suites. Qemu is installed always into the same directory. This
patch changes the default behavior to install qemu over and install
pathced version to separate directory. It also disables force install.

Change-Id: I0d7493a02b026a6ae4a5ea8bacf54656de9db567
Signed-off-by: Peter Mikus <pmikus@cisco.com>

6 files changed, 89 insertions, 99 deletions

diff --git a/resources/libraries/bash/config/config b/resources/libraries/bash/config/config
index 2b71edcc49..42f23bdaf8 100644
--- a/resources/libraries/bash/config/config
+++ b/resources/libraries/bash/config/config
@@ -1,5 +1,8 @@
-QEMU_INSTALL_DIR=/opt
+QEMU_INSTALL_DIR=/opt/qemu-2.5.0
 QEMU_INSTALL_VERSION=qemu-2.5.0
+QEMU_PATCH=false
+QEMU_FORCE_INSTALL=false
+QEMU_TARGET_LIST=x86_64-softmmu
 
 DPDK_INSTALL_DIR=/opt
-DPDK_INSTALL_VERSION=dpdk-17.11
\ No newline at end of file
+DPDK_INSTALL_VERSION=dpdk-17.11
diff --git a/resources/libraries/bash/config/defaults b/resources/libraries/bash/config/defaults
index 1547452982..a70add12b1 100644
--- a/resources/libraries/bash/config/defaults
+++ b/resources/libraries/bash/config/defaults
@@ -2,11 +2,14 @@
 
 typeset -A cfg
 cfg=( # set default values in config array
-    [QEMU_INSTALL_DIR]="/opt"
+    [QEMU_INSTALL_DIR]="/opt/qemu-2.5.0"
     [QEMU_INSTALL_VERSION]="qemu-2.5.0"
+    [QEMU_PATCH]=false
+    [QEMU_FORCE_INSTALL]=false
+    [QEMU_TARGET_LIST]=x86_64-softmmu
     [DPDK_INSTALL_DIR]=/opt
     [DPDK_INSTALL_VERSION]=dpdk-17.11
     [K8S_CALICO]="${SCRIPT_DIR}/../../templates/kubernetes/calico_v2.6.3.yaml"
     [K8S_CONTIV_VPP]="https://raw.githubusercontent.com/contiv/vpp/master/k8s/contiv-vpp.yaml"
     [K8S_CSIT]="${SCRIPT_DIR}/../../templates/kubernetes/csit.yaml"
-)
\ No newline at end of file
+)
diff --git a/resources/libraries/bash/qemu_build.sh b/resources/libraries/bash/qemu_build.sh
index 57520a9b5e..4638ec1bcb 100755
--- a/resources/libraries/bash/qemu_build.sh
+++ b/resources/libraries/bash/qemu_build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -12,81 +12,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-QEMU_VERSION="qemu-2.5.0"
-QEMU_DOWNLOAD_REPO="http://download.qemu-project.org/"
-QEMU_DOWNLOAD_PACKAGE="${QEMU_VERSION}.tar.xz"
-QEMU_PACKAGE_URL="${QEMU_DOWNLOAD_REPO}${QEMU_DOWNLOAD_PACKAGE}"
-QEMU_INSTALL_DIR="/opt/${QEMU_VERSION}"
+set -x
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TARGET_LIST="x86_64-softmmu"
 
+# Include
+source ${SCRIPT_DIR}/config/defaults
+source ${SCRIPT_DIR}/shell/qemu_utils.sh
+
+# Read configuration
+while read line
+do
+    if echo $line | grep -F = &>/dev/null
+    then
+        varname=$(echo "$line" | cut -d '=' -f 1)
+        cfg[$varname]=$(echo "$line" | cut -d '=' -f 2-)
+    fi
+done < ${SCRIPT_DIR}/config/config
+
+# Read parameters
 for i in "$@"; do
     case $i in
         --version=*)
-            QEMU_VERSION="${i#*=}"
+            cfg['QEMU_INSTALL_VERSION']="${i#*=}"
             shift ;;
         --directory=*)
-            QEMU_INSTALL_DIR="${i#*=}"
+            cfg['QEMU_INSTALL_DIR']="${i#*=}"
             shift ;;
         --patch)
-            PATCH=1
+            cfg['QEMU_PATCH']=true
             shift ;;
         --force)
-            FORCE=1
+            cfg['QEMU_FORCE_INSTALL']=true
             shift ;;
         --target-list)
-            TARGET_LIST="${i#*=}"
+            cfg['QEMU_TARGET_LIST']="${i#*=}"
             shift ;;
         *)
             ;;
     esac
 done
 
-if test "$(id -u)" -ne 0
-then
-    echo "Please use root or sudo to be able to install into: ${QEMU_INSTALL_DIR}"
-    exit 1
-fi
-
-WORKING_DIR=$(mktemp -d) || \
-    { echo "Failed to create temporary working dir"; exit 1; }
-trap "rm -r ${WORKING_DIR}" EXIT
-
-if [ $FORCE ]
-then
-    rm -rf ${QEMU_INSTALL_DIR}
-else
-    test -d ${QEMU_INSTALL_DIR} && \
-        { echo "Qemu already installed: ${QEMU_INSTALL_DIR}"; exit 0; }
-fi
-
-# Download QEMU source code if no local copy exists
-if [ ! -f /opt/${QEMU_DOWNLOAD_PACKAGE} ]; then
-    wget -P /opt -q ${QEMU_PACKAGE_URL} || \
-        { echo "Failed to download ${QEMU_VERSION}"; exit 1; }
-fi
-
-# Extract archive into temp directory
-tar --strip-components 1 -xf /opt/${QEMU_DOWNLOAD_PACKAGE} -C ${WORKING_DIR} || \
-    { echo "Failed to extract ${QEMU_VERSION}.tar.xz"; exit 1; }
-
-cd ${WORKING_DIR}
-mkdir ${QEMU_INSTALL_DIR} || \
-    { echo "Failed to create ${QEMU_INSTALL_DIR}"; exit 1; }
-
-# Apply additional patches
-if [ $PATCH ]
-then
-    chmod +x ${SCRIPT_DIR}/qemu_patches/${QEMU_VERSION}/*
-    run-parts --verbose --report  ${SCRIPT_DIR}/qemu_patches/${QEMU_VERSION}
-fi
-
-# Build
-./configure --target-list=${TARGET_LIST} --prefix=${QEMU_INSTALL_DIR} || \
-    { echo "Failed to configure ${QEMU_VERSION}"; exit 1; }
-make -j`nproc` || \
-    { echo "Failed to compile ${QEMU_VERSION}"; exit 1; }
-make install || \
-    { echo "Failed to install ${QEMU_VERSION}"; exit 1; }
-
-echo QEMU ${QEMU_VERSION} ready
+# Install qemu
+qemu_utils.qemu_install ${cfg[QEMU_INSTALL_DIR]} ${cfg[QEMU_INSTALL_VERSION]} \
+    ${cfg[QEMU_PATCH]} ${cfg[QEMU_FORCE_INSTALL]} ${cfg[QEMU_TARGET_LIST]}
diff --git a/resources/libraries/bash/shell/qemu_utils.sh b/resources/libraries/bash/shell/qemu_utils.sh
index 510d9f2838..cabf93d978 100644
--- a/resources/libraries/bash/shell/qemu_utils.sh
+++ b/resources/libraries/bash/shell/qemu_utils.sh
@@ -15,47 +15,47 @@
 function qemu_utils.qemu_delete {
     # Deletes the QEMU directory
     # QEMU install directory
-    qemu_install_dir=$1
-    # QEMU install version
-    qemu_install_ver=$2
+    local qemu_install_dir=$1
 
-    [ -d ${qemu_install_dir}/${qemu_install_ver} ] && \
-        sudo rm -r ${qemu_install_dir}/${qemu_install_ver} && \
-        echo "${qemu_install_dir}/${qemu_install_ver} removed"
+    [ -d ${qemu_install_dir} ] && \
+        sudo rm -r ${qemu_install_dir} && \
+        echo "${qemu_install_dir} removed"
 }
 
 function qemu_utils.qemu_install {
     # Downloads and installs QEMU
     # QEMU install directory
-    qemu_install_dir=$1
+    local qemu_install_dir=$1
     # QEMU install version
-    qemu_install_ver=$2
+    local qemu_install_ver=$2
     # QEMU patch
-    qemu_patch=$3
+    local qemu_patch=${3:-false}
     # Force install (if true then remove previous installation; default false)
-    force_install=${4:-false}
+    local force_install=${4:-false}
     # QEMU repo URL
-    qemu_package_url="http://download.qemu-project.org/${qemu_install_ver}.tar.xz"
+    local qemu_package_url="http://download.qemu-project.org/${qemu_install_ver}.tar.xz"
+    # QEMU target arch
+    local qemu_target_list=${5:-x86_64-softmmu}
 
-    if [ $force_install ]; then
+    if [ $force_install = true ]; then
         # Cleanup QEMU dir
-        qemu_utils.qemu_delete $qemu_install_dir $qemu_install_ver
+        qemu_utils.qemu_delete $qemu_install_dir
     else
         # Test if QEMU was installed previously
         test -d $qemu_install_dir && \
             { echo "Qemu already installed: $qemu_install_dir"; exit 0; }
     fi
 
-    tmp_dir=$(mktemp -d) || \
+    local tmp_dir=$(mktemp -d) || \
         { echo "Failed to create temporary working dir"; exit 1; }
-    trap "rm -r ${tmp_dir}" EXIT
+    trap "sudo rm -r ${tmp_dir}" EXIT
 
     # Download QEMU source code if no local copy exists
     if [ ! -f /opt/${qemu_install_ver}.tar.xz ]; then
         sudo wget -e use_proxy=yes -P /opt -q ${qemu_package_url} || \
             { echo "Failed to download ${qemu_install_ver}"; exit 1; }
     fi
-    tar --strip-components 1 -xvJf ${tmp_dir}/${qemu_install_ver}.tar.xz -C ${tmp_dir} && \
+    tar --strip-components 1 -xf /opt/${qemu_install_ver}.tar.xz -C ${tmp_dir} || \
         { echo "Failed to exctract ${qemu_install_ver}.tar.xz"; exit 1; }
 
     cd ${tmp_dir}
@@ -63,14 +63,14 @@ function qemu_utils.qemu_install {
         { echo "Failed to create ${qemu_install_dir}"; exit 1; }
 
     # Apply additional patches
-    if [ $qemu_patch ]
+    if [ $qemu_patch = true ]
     then
         chmod +x ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver}/*
         run-parts --verbose --report  ${SCRIPT_DIR}/qemu_patches/${qemu_install_ver}
     fi
 
     # Build
-    sudo ./configure --target-list=x86_64-softmmu --prefix=${qemu_install_dir}/${qemu_install_ver} || \
+    sudo ./configure --target-list=${qemu_target_list} --prefix=${qemu_install_dir} || \
         { echo "Failed to configure ${qemu_install_ver}"; exit 1; }
     sudo make -j`nproc` || \
         { echo "Failed to compile ${qemu_install_ver}"; exit 1; }
@@ -78,4 +78,4 @@ function qemu_utils.qemu_install {
         { echo "Failed to install ${qemu_install_ver}"; exit 1; }
 
     echo "QEMU ${qemu_install_ver} ready"
-}
\ No newline at end of file
+}
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index a8c26d6898..42ccb8c9dd 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -688,6 +688,10 @@ class QemuUtils(object):
         ssh.connect(node)
 
         directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR)
+        if apply_patch:
+            directory += '-patch'
+        else:
+            directory += '-base'
         version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION)
         force = ' --force' if force_install else ''
         patch = ' --patch' if apply_patch else ''
diff --git a/resources/libraries/robot/performance/performance_configuration.robot b/resources/libraries/robot/performance/performance_configuration.robot
index 61bc4bd3a4..e544cdd3a3 100644
--- a/resources/libraries/robot/performance/performance_configuration.robot
+++ b/resources/libraries/robot/performance/performance_configuration.robot
@@ -1465,8 +1465,11 @@
 | | ... | jumbo_frames=${jumbo_frames}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1
@@ -1516,8 +1519,6 @@
 | | | ... | skip=${skip_cpus} | count=${vm_cpus} | qemu_id=${number}
 | | | ... | jumbo_frames=${jumbo_frames}
 | | | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM${number} | ${vm2}
-| | | Run Keyword Unless | ${qemu_built} | Set Suite Variable | ${qemu_built}
-| | | ... | ${True}
 
 | Configure guest VM with dpdk-testpmd using SMT connected via vhost-user
 | | [Documentation]
@@ -1558,8 +1559,11 @@
 | | ... | jumbo_frames=${jumbo_frames}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1
@@ -1629,8 +1633,11 @@
 | | ... | jumbo_frames=${jumbo_frames}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1
@@ -1688,8 +1695,6 @@
 | | | ... | count=${vm_cpus} | qemu_id=${number}
 | | | ... | jumbo_frames=${jumbo_frames}
 | | | Set To Dictionary | ${dut2_vm_refs} | DUT2_VM${number} | ${vm2}
-| | | Run Keyword Unless | ${qemu_built} | Set Suite Variable | ${qemu_built}
-| | | ... | ${True}
 
 | Configure guest VM with dpdk-testpmd-mac using SMT connected via vhost-user
 | | [Documentation]
@@ -1733,8 +1738,11 @@
 | | ... | jumbo_frames=${jumbo_frames}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1
@@ -1783,8 +1791,11 @@
 | | Run keyword | ${vm_name}.Qemu Add Vhost User If | ${sock2}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1
@@ -1833,8 +1844,11 @@
 | | Run keyword | ${vm_name}.Qemu Add Vhost User If | ${sock2}
 | | ${apply_patch}= | Set Variable If | "${perf_qemu_qsz}" == "256" | ${False}
 | | ... | ${True}
-| | Run Keyword Unless | ${qemu_built} | ${vm_name}.Build QEMU | ${dut_node}
-| | ... | force_install=${True} | apply_patch=${apply_patch}
+| | ${perf_qemu_path}= | Set Variable If | ${apply_patch}
+| | ... | ${perf_qemu_path}-patch/bin/
+| | ... | ${perf_qemu_path}-base/bin/
+| | Run Keyword If | ${qemu_build} | ${vm_name}.Build QEMU | ${dut_node}
+| | ... | apply_patch=${apply_patch}
 | | Run keyword | ${vm_name}.Qemu Set Path | ${perf_qemu_path}
 | | Run keyword | ${vm_name}.Qemu Set Node | ${dut_node}
 | | Run keyword | ${vm_name}.Qemu Set Smp | ${count} | ${count} | 1 | 1