diff options
Diffstat (limited to 'vhost-test/vhost.sh')
| -rwxr-xr-x | vhost-test/vhost.sh | 541 |
1 files changed, 541 insertions, 0 deletions
diff --git a/vhost-test/vhost.sh b/vhost-test/vhost.sh new file mode 100755 index 0000000..c80e39f --- /dev/null +++ b/vhost-test/vhost.sh @@ -0,0 +1,541 @@ +#!/bin/bash -e + +CD="$( cd "$( dirname $0 )" && pwd )" + +CONFIG_FILE="" + +TMP_DIR="${CD}/tmp/" +VMDIR="${CD}/tmp/vmdir/" +VMMOUNT="${CD}/vmroot/" +VMWORK="${CD}/tmp/work/" + +BRNAME="vhtestbr0" +VMTAP="vhtesttap0" +VPPSCREEN="vhtestvpp" + +VM_ROOT="/" +VM_VNCPORT="3555" + +# Those variables are found after parsing the conf +VPP_BUILD="xxx" +VPP_INSTALL="xxx" +VPP="xxx" +DPDK_BIND="xxx" + +CORES_VM_LIST="xxx" +CORES_VM_N="xxx" +declare -a CORES_VM_ARRAY +CORES_VPP_LIST="xxx" +CORES_VPP_N="xxx" +declare -a CORES_VPP_ARRAY + +function validate_parameter() { + for c in $@; do + [ "${!c}" = "" ] && echo "Configuration paramater $c is not set in $CONFIG_FILE" && return 1 + done + return 0 +} + +function validate_directory() { + for c in $@; do + [ ! -d "${!c}" ] && echo "$c=${!c} is not a directory" && return 1 + done + return 0 +} + +function validate_file() { + for c in $@; do + [ ! -f "${!c}" ] && echo "$c=${!c} is not a file" && return 1 + done + return 0 +} + +function validate_exec() { + for c in $@; do + [ ! -e "${!c}" -a "$(echo ${!c} | grep '/')" = "" -a "$(which ${!c})" != "" ] && eval $c=$(which ${!c}) + [ ! -x "${!c}" ] && echo "$c=${!c} is not executable" && return 1 + done + return 0 +} + +function validate_cores() { + for c in $@; do + LIST_NAME="${c}_LIST" + N_NAME="${c}_N" + ARRAY_NAME="${c}_ARRAY" + eval $LIST_NAME=\"$(echo ${!c} | sed 's/,/ /g')\" + COUNT=0 + for cid in ${!LIST_NAME}; do + if ! [[ "$cid" =~ ^[0-9]+$ ]]; then + echo "'$cid' is not a valid core ID" + return 1 + fi + eval $ARRAY_NAME[$COUNT]=$cid + COUNT=$(expr $COUNT + 1) + done + eval $N_NAME=$COUNT + #echo $LIST_NAME=${!LIST_NAME} + #echo $N_NAME=${!N_NAME} + done +} + +function install_directory() { + for c in $@; do + echo "mkdir ${!c}" + [ ! -d "${!c}" ] && mkdir -p ${!c} + done + for c in $@; do + [ ! -d "${!c}" ] && return 1 + done + return 0 +} + +function load_config() { + if [ "$CONFIG_FILE" != "" ]; then + CONFIG_FILE="$CONFIG_FILE" + elif [ -f "${CD}/conf.sh" ]; then + CONFIG_FILE="${CD}/conf.sh" + else + CONFIG_FILE="${CD}/conf.sh.default" + fi + . $CONFIG_FILE + + #Validate config + validate_parameter VPP_DIR VPP_IF0_PCI VPP_IF0_MAC VPP_IF1_PCI VPP_IF1_MAC VPP_IF0_NAME VPP_IF1_NAME \ + QEMU VM_ROOT VM_INITRD VM_VMLINUZ VM_VNCPORT VM_USERNAME + validate_directory VPP_DIR VM_ROOT + validate_file VM_INITRD VM_VMLINUZ + validate_exec QEMU + validate_cores CORES_VM CORES_VPP + + [ ! -d "$VPP_DIR/vnet" ] && echo "VPP_DIR=$VPP_DIR is not VPP source directory" && exit 5 + [ "$($QEMU --version | grep QEMU)" = "" ] && echo "$QEMU is probably not a qemu executable" && exit 6 + + if [ "$VPP_BUILD" = "release" ] ; then + VPP_INSTALL="$VPP_DIR/build-root/install-vpp-native/" + VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/" + elif [ "$VPP_BUILD" = "debug" ] ; then + VPP_INSTALL="$VPP_DIR/build-root/install-vpp_debug-native/" + VPP_BUILD="$VPP_DIR/build-root/build-vpp-native/" + else + echo "Invalid VPP_BUILD parameter '$VPP_BUILD'" && exit 1 + fi + + if [ "$QUEUES" != "1" -a "$QUEUES" != "2" ]; then + echo "QUEUES can only be 1 or 2" + exit 7 + fi + + VPP="$VPP_INSTALL/vpp/bin/vpp" + DPDK_BIND="$(ls $VPP_BUILD/dpdk/dpdk-*/tools/dpdk-devbind.py | head -n 1)" + + validate_exec VPP DPDK_BIND + + return 0 +} + +function banner() { + echo "-------------------------------------" + echo " VPP vhost test - BETA" + echo "-------------------------------------" + echo "VPP_DIR : $VPP_DIR" + echo "VPP : $VPP" + echo "DPDK_BIND : $DPDK_BIND" + echo "Qemu : $QEMU" + echo "Qemu : $($QEMU --version)" + echo "VM cores : ${CORES_VM_ARRAY[*]}" + echo "VPP cores : ${CORES_VPP_ARRAY[*]}" + echo "-------------------------------------" +} + +function vmdir_umount() { + sleep 0.5 + sudo umount $VMMOUNT + sleep 0.5 + rmdir "$VMMOUNT" +} + +function vmdir_mount() { + mkdir -p "$VMDIR" + mkdir -p "$VMMOUNT" + mkdir -p "$VMWORK" + sudo mount -t overlayfs -o lowerdir=${VM_ROOT},workdir=${VMWORK},upperdir=${VMDIR} overlayfs ${VMMOUNT} +} + +function clean() { + #Cleaning + vmdir_umount > /dev/null 2>&1 || echo -n "" #Just make sure it's not running + if [ ! -d "$TMP_DIR" ]; then + echo "$TMP_DIR" + mkdir -p "$TMP_DIR" + touch "$TMP_DIR/.vpp-vhost-test-safety" + elif [ ! -f "$TMP_DIR/.vpp-vhost-test-safety" ]; then + echo "Error: I will not remove tmp directory as there is no safety file: $TMP_DIR/.vpp-vhost-test-safety" + echo "Please do 'touch $TMP_DIR/.vpp-vhost-test-safety' if you are sure the content of this directory can be removed" + exit 7 + else + sudo rm -rf $TMP_DIR/ + mkdir -p "$TMP_DIR" + touch "$TMP_DIR/.vpp-vhost-test-safety" + fi +} + +function prepare_testpmd() { + #Set the VM in testpmd mode + cat > "$VMDIR/etc/startup.d/testpmd.sh" << EOF +#!/bin/sh +sysctl -w vm.nr_hugepages=1024 +mkdir -p /mnt/huge +mount -t hugetlbfs none /mnt/huge +modprobe uio +insmod ${VPP_INSTALL}/dpdk/kmod/igb_uio.ko +$DPDK_BIND -b igb_uio 00:07.0 +$DPDK_BIND -b igb_uio 00:08.0 +#gdb -ex run --args +screen -d -m ${VPP_INSTALL}/dpdk/app/testpmd -l 0,1,2,3,4 --master-lcore 0 --socket-mem 512 --proc-type auto --file-prefix pg -w 0000:00:07.0 -w 0000:00:08.0 -- --disable-hw-vlan --rxq=$QUEUES --txq=$QUEUES --rxd=256 --txd=256 --auto-start --nb-cores=4 --eth-peer=0,aa:aa:aa:aa:bb:b1 --eth-peer=1,aa:aa:aa:aa:bb:b2 --port-topology=chained +#--log-level 10 +#--rxq=2 --txq=2 + +for i in \$(ls /proc/irq/ | grep [0-9]); do echo 1 > /proc/irq/\$i/smp_affinity ; done +echo "0" > /proc/sys/kernel/watchdog_cpumask +EOF +sudo chmod u+x "$VMDIR/etc/startup.d/testpmd.sh" +} + +function prepare_vm() { + #Generate VM configuration files in $VMDIR + mkdir -p "$VMDIR/etc/network" + echo "vpp-vhost-test-vm" > "$VMDIR/etc/hostname" + + cat > "$VMDIR/etc/hosts" << EOF +127.0.0.1 localhost.localdomain localhost +127.0.1.1 $vpp-vhost-test-vm +EOF + + cat > "$VMDIR/etc/rc.local" << EOF +#!/bin/sh +mkdir -p /var/log/startup/ +for exe in \`ls /etc/startup.d\`; do + echo -n "Startup script \$exe " + ( (nohup /etc/startup.d/\$exe > /var/log/startup/\$exe 2>&1 &) && echo "[OK]") || echo "[Failed]" +done +exit 0 +EOF + sudo chmod a+x "$VMDIR/etc/rc.local" + + mkdir -p $VMDIR/etc/udev/rules.d/ + cat > "$VMDIR/etc/udev/rules.d/70-persistent-net.rules" << EOF +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:00:00:10:10:10", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0" +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:00", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost0" +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="de:ad:be:ef:01:01", ATTR{type}=="1", KERNEL=="eth*", NAME="vhost1" +EOF + cat > "$VMDIR/etc/fstab" << EOF +/dev/sda1 / ext4 errors=remount-ro 0 1 +EOF + + cat > "$VMDIR/etc/network/interfaces" << EOF +auto lo +iface lo inet loopback +auto eth0 +iface eth0 inet manual +iface eth0 inet6 static + address fd00::1 + netmask 64 +EOF + + mkdir -p "$VMDIR/etc/startup.d" + cat > "$VMDIR/etc/startup.d/dmesg.sh" << EOF +#!/bin/sh +while [ "1" = "1" ]; do + dmesg > /var/log/startup/dmesg + sleep 10 + echo "--------------------" +done +EOF + chmod u+x $VMDIR/etc/startup.d/*.sh + mkdir -p "$VMDIR/etc/default" + cat > "$VMDIR/etc/default/irqbalance" << EOF +ENABLED="0" +ONESHOT="0" +EOF + + prepare_testpmd + + MQ="" + if [ "$QUEUES" != "1" ]; then + MQ=",mq=on" + fi + LAST_VM_CPU="$(expr $CORES_VM_N - 1)" + + cat << EOF > "$TMP_DIR/vm.conf" +-enable-kvm -machine pc -initrd $VM_INITRD -kernel $VM_VMLINUZ -vnc 127.0.0.1:1 -m 4G +-append 'root=ro ro rootfstype=9p rootflags=trans=virtio nohz_full=1-$LAST_VM_CPU isolcpus=1-$LAST_VM_CPU rcu_nocbs=1-$LAST_VM_CPU selinux=0 audit=0 net.ifnames=0 biosdevname=0' +-cpu host -smp $CORES_VM_N +-device e1000,netdev=network0,mac=00:00:00:10:10:10,bus=pci.0,addr=3.0 +-netdev tap,id=network0,ifname=$VMTAP,script=no,downscript=no +-fsdev local,security_model=none,id=fsdev_id,path=${VMMOUNT} +-device virtio-9p-pci,id=dev_fs,fsdev=fsdev_id,mount_tag=ro +-daemonize -pidfile $TMP_DIR/qemu.pid + +-chardev socket,id=chr0,path=$TMP_DIR/sock0,server +-netdev type=vhost-user,id=thrnet0,chardev=chr0,queues=$QUEUES +-device virtio-net-pci,netdev=thrnet0,mac=de:ad:be:ef:01:00,bus=pci.0,addr=7.0${MQ} +-object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on +-numa node,memdev=mem +-chardev socket,id=chr1,path=$TMP_DIR/sock1,server +-netdev type=vhost-user,id=thrnet1,chardev=chr1,queues=$QUEUES +-device virtio-net-pci,netdev=thrnet1,mac=de:ad:be:ef:01:01,bus=pci.0,addr=8.0${MQ} +EOF +} + +function prepare_vpp() { + VPP_CPU="" + VPP_DEV0="" + VPP_DEV1="" + ENABLE_MULTIQUEUE="" + if [ "$CORES_VPP_N" = "0" ]; then + VPP_CPU="" + elif [ "$CORES_VPP_N" = "1" ]; then + VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]}" + elif [ "$CORES_VPP_N" -lt "4" ]; then + VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]}" + VPP_DEV0="workers 0" + VPP_DEV1="workers 1" + else + VPP_CPU="corelist-workers ${CORES_VPP_ARRAY[0]},${CORES_VPP_ARRAY[1]},${CORES_VPP_ARRAY[2]},${CORES_VPP_ARRAY[3]}" + VPP_DEV0="workers 0,1" + VPP_DEV1="workers 2,3" + ENABLE_MULTIQUEUE="1" + fi + + if [ "$QUEUES" = "2" -a "$ENABLE_MULTIQUEUE" = "1" ]; then + VPP_DEV0="$VPP_DEV0 num-rx-queues 2 num-tx-queues 2" + VPP_DEV1="$VPP_DEV1 num-rx-queues 2 num-tx-queues 2" + fi + + cat << EOF > "$TMP_DIR/vpp.cmdline" +cpu { $VPP_CPU } +unix { startup-config $TMP_DIR/vpp.conf interactive } +dpdk { dev $VPP_IF0_PCI { $VPP_DEV0 } dev $VPP_IF1_PCI { $VPP_DEV1 } } +EOF + + cat << EOF > "$TMP_DIR/vpp.conf" +create vhost-user socket $TMP_DIR/sock0 hwaddr aa:aa:aa:aa:bb:b1 +create vhost-user socket $TMP_DIR/sock1 hwaddr aa:aa:aa:aa:bb:b2 +set interface l2 xconnect VirtualEthernet0/0/0 $VPP_IF0_NAME +set interface l2 xconnect VirtualEthernet0/0/1 $VPP_IF1_NAME +set interface l2 xconnect $VPP_IF0_NAME VirtualEthernet0/0/0 +set interface l2 xconnect $VPP_IF1_NAME VirtualEthernet0/0/1 +set interface state VirtualEthernet0/0/0 up +set interface state VirtualEthernet0/0/1 up +set interface state $VPP_IF1_NAME up +set interface state $VPP_IF0_NAME up +EOF + + #VHOST queue pining + if [ "$QUEUES" = "1" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then + if [ "$CORES_VPP_N" = "0" ]; then + echo -n "" + elif [ "$CORES_VPP_N" = "1" ]; then + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 1 +vhost thread VirtualEthernet0/0/0 1 +EOF + elif [ "$CORES_VPP_N" -lt "4" ]; then + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 2 +vhost thread VirtualEthernet0/0/0 1 +EOF + else + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 3 +vhost thread VirtualEthernet0/0/0 4 +EOF + fi + elif [ "$QUEUES" = "2" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then + if [ "$CORES_VPP_N" = "0" ]; then + echo -n "" + elif [ "$CORES_VPP_N" = "1" ]; then + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 1 +vhost thread VirtualEthernet0/0/1 1 +vhost thread VirtualEthernet0/0/0 1 +vhost thread VirtualEthernet0/0/0 1 +EOF + elif [ "$CORES_VPP_N" -lt "4" ]; then + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 2 +vhost thread VirtualEthernet0/0/1 2 +vhost thread VirtualEthernet0/0/0 1 +vhost thread VirtualEthernet0/0/0 1 +EOF + else + cat << EOF >> "$TMP_DIR/vpp.conf" +vhost thread VirtualEthernet0/0/1 3 +vhost thread VirtualEthernet0/0/1 4 +vhost thread VirtualEthernet0/0/0 1 +vhost thread VirtualEthernet0/0/0 2 +EOF + fi + fi + +} + +function prepare() { + #Generate all configuration and VM files + clean + prepare_vm + prepare_vpp +} + +function start_vpp() { + GDB="" + if [ "$VPP_GDB" = "1" ]; then + [ -e "$TMP_DIR/vpp.sh.gdbinit" ] && sudo rm "$TMP_DIR/vpp.sh.gdbinit" + cat << EOF > "$TMP_DIR/vpp.sh.gdbinit" +handle SIGUSR1 nostop +handle SIGUSR1 noprint +set print thread-events off +run +EOF + GDB="gdb -x $TMP_DIR/vpp.sh.gdbinit --args " + fi + + echo "------- Starting VPP --------" + echo " Screen $VPPSCREEN (sudo screen -r $VPPSCREEN)" + echo " Command-line Conf:" + cat $TMP_DIR/vpp.cmdline + echo " CLI Conf:" + cat $TMP_DIR/vpp.conf + echo "-----------------------------" + + sudo screen -d -m -S "$VPPSCREEN" $GDB $VPP_DIR/build-root/install-vpp-native/vpp/bin/vpp -c $TMP_DIR/vpp.cmdline +} + +function start_vm() { + echo "------- Starting VM --------" + echo " VM conf:" + cat $TMP_DIR/vm.conf + echo "----------------------------" + + #Eval is used so that ' characters are not ignored + eval sudo chrt -rr 1 taskset -c $CORES_VM $QEMU $(cat $TMP_DIR/vm.conf) + + echo "Started QEMU with PID $(sudo cat $TMP_DIR/qemu.pid)" + + sudo brctl addbr $BRNAME + sudo ip link set $BRNAME up + sudo ip addr add fd00::ffff/64 dev $BRNAME + sudo brctl addif $BRNAME $VMTAP + sudo ip link set $VMTAP up +} + +function start() { + if [ -f "$TMP_DIR/.started" ]; then + echo "$TMP_DIR/.started exists" + echo "This means the setup is probably running already." + echo "Please stop the setup first, or remove this file." + exit 2 + fi + + banner + + prepare + + touch "$TMP_DIR/.started" + + start_vpp + + vmdir_mount + + start_vm +} + +function pin_vm() { + PIDS=$(ps -eLf | grep qemu-system-x86_64 | awk '$5 > 50 { print $4; }') + idx=1 + for p in $PIDS; do + if [ "${CORES_VM_ARRAY[$idx]}" = "" ]; then + echo "Too many working threads in VM" + return 1 + fi + echo "VM PID $p on core ${CORES_VM_ARRAY[$idx]}" + sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p + idx=$(expr $idx + 1) + done +} + +function pin_vpp() { + PIDS=$(ps -eLf | grep /bin/vpp | awk '$5 > 50 { print $4; }') + idx=0 + for p in $PIDS; do + if [ "${CORES_VPP_ARRAY[$idx]}" = "" ]; then + echo "Too many working threads in VPP" + return 1 + fi + echo "VPP PID $p on core ${CORES_VPP_ARRAY[$idx]}" + sudo taskset -pc ${CORES_VPP_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p + idx=$(expr $idx + 1) + done +} + +function pin() { + pin_vm + pin_vpp +} + +function stop() { + set +e + + [ -f "$TMP_DIR/qemu.pid" ] && echo "Stopping VM ($(sudo cat $TMP_DIR/qemu.pid))" && sudo kill "$(sudo cat $TMP_DIR/qemu.pid)" && sudo rm $TMP_DIR/qemu.pid + + vmdir_umount + + sudo ip link set $BRNAME down + sudo brctl delbr $BRNAME + + sudo screen -S "$VPPSCREEN" -X quit && echo "Stopping VPP" + + [ -f "$TMP_DIR/.started" ] && rm "$TMP_DIR/.started" +} + +function cmd_openvnc() { + load_config + echo Please VNC to 5900 to connect to this VM console + socat TCP6-LISTEN:5900,reuseaddr TCP:localhost:5901 & +} + +function cmd_start() { + load_config + start +} + +function cmd_pin() { + load_config + pin +} + +function cmd_stop() { + load_config + stop +} + +function cmd_clean() { + load_config + clean +} + +function cmd_ssh() { + load_config + ssh ${VM_USERNAME}@fd00::1 +} + +function cmd_config() { + load_config +} + +[ "$1" = "" ] && echo "Missing arguments" && usage && exit 1 +CMD="$1" +shift +eval "cmd_$CMD" "$@" + |