diff options
| -rw-r--r-- | vhost-test/.gitignore | 3 | ||||
| -rw-r--r-- | vhost-test/README.md | 29 | ||||
| -rw-r--r-- | vhost-test/conf.sh.default | 4 | ||||
| -rw-r--r-- | vhost-test/mg.lua | 164 | ||||
| -rwxr-xr-x | vhost-test/vhost.sh | 300 |
5 files changed, 412 insertions, 88 deletions
diff --git a/vhost-test/.gitignore b/vhost-test/.gitignore new file mode 100644 index 0000000..2272b59 --- /dev/null +++ b/vhost-test/.gitignore @@ -0,0 +1,3 @@ +tmp +vmroot +conf.sh diff --git a/vhost-test/README.md b/vhost-test/README.md index ee5f56f..5564b18 100644 --- a/vhost-test/README.md +++ b/vhost-test/README.md @@ -52,11 +52,30 @@ Finally, when you are done, you can stop the VMs. $ ./vhost.sh stop -## Traffic Generation - -Traffic generation is, for now, out of the scope of this script. -You are supposed to update ./conf.sh by setting up the right parameters. -Use the traffic generator you like to test the perfs. +## Traffic Generation with MoonGen + +mg.lua is intended to be used with the MoonGen packet generator. +This script measures packet loss and forwarding with a user-defined +granularity. The script keeps running while consecutive measures are +too far on both Tx axis (--maxRateInterval option) or on the drop rate +axis (--targetDropRateRatio). The latter is a ratio instead of an interval, +as the packet drop is mostly a logarithmic value. + +The script is used as follows: +sudo /path/to/MoonGen ./mg.lua <options> + +Mandatory options are: +--rxport <id> MoonGen's rx port index +--txport <id> MoonGen's tx port index +--dst <hwaddr> Frame's destination L2 address + +Other options are: +--duration <seconds> Each measurement duration +--frameSize <bytes> Each frame size +--maxRateInterval <%> Max Tx interval between measure +--targetDropRateRatio <ratio> Max ratio between two drop rate measures +--minRateInterval <%> Min Tx interval (Will override drop ratio in case of non-continuous) +--out <file> Output file ## Administrativa diff --git a/vhost-test/conf.sh.default b/vhost-test/conf.sh.default index b3870d8..fd880d1 100644 --- a/vhost-test/conf.sh.default +++ b/vhost-test/conf.sh.default @@ -56,3 +56,7 @@ VM_VMLINUZ="/boot/vmlinuz-$(uname -r)" # to the running VM VM_USERNAME="$USER" +# Set custom queue size (default is 256). +# Setting a different value (512 or 1024) requires a patched qemu. +VIRTIO_QSZ="256" + diff --git a/vhost-test/mg.lua b/vhost-test/mg.lua new file mode 100644 index 0000000..21c4afa --- /dev/null +++ b/vhost-test/mg.lua @@ -0,0 +1,164 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local stats = require "stats" +local hist = require "histogram" +local timer = require "timer" +local barrier = require "barrier" +local table = require "table" +local math = require "math" + +function configure(parser) + parser:description("Packet loss graph.") + parser:option("--txport", "Device to transmit/receive from."):convert(tonumber) + parser:option("--rxport", "Device to transmit/receive from."):convert(tonumber) + parser:option("--dst", "Frame's destination hardware address") + parser:option("--duration", "Device to transmit/receive from."):default(20):convert(tonumber) + parser:option("--frameSize", "Ethernet frame size."):default(64):convert(tonumber) + parser:option("--maxRateInterval", "Max % rate interval between measures"):default(5):convert(tonumber) + parser:option("--minRateInterval", "Min % rate interval between measures"):default(0.1):convert(tonumber) + parser:option("--targetDropRateRatio", "Target drop rate ratio between measures"):default(2):convert(tonumber) + parser:option("--out", "Output file"):default("vhost_mg_"..os.date("%F_%H-%M")..".txt") +end + +function getHeaderString(file) + return string.format("%10s\t%10s\t%12s\t%12s\t%12s\t%12s\t%12s\t%12s\t%12s\t%12s", + "size(B)", "Time(s)", "Rate(Mbps)", "Tx(pkt)", "Rx(pkt)", "DropRate(%)", "Tx(Mpps)", "Rx(Mpps)", "Tx(Gbps)", "Rx(Gbps)") +end + +function getResultString(result, file) + return string.format("%10d\t%10d\t%12f\t%12d\t%12d\t%12f\t%12f\t%12f\t%12f\t%12f", + result.frameSize, result.duration, result.rate, result.tx, result.rx, result.dropRate, result.txMpps, result.rxMpps, result.txRate, result.rxRate) +end + +function runMeasure(txDev, rxDev, frameSize, duration, rate) + local bar = barrier.new(2) + local result = {} + txDev:getTxQueue(0):setRate(rate * frameSize / (frameSize+20)) + local stask = mg.startTask("loadSlave", txDev:getTxQueue(0), frameSize, duration, bar) + local rtask = mg.startTask("counterSlave", rxDev:getRxQueue(0), frameSize, duration, bar) + result.frameSize = frameSize + result.duration = duration + result.tx = stask:wait() + result.rx = rtask:wait() + result.rate = rate + result.dropRate = (result.tx - result.rx)/result.tx * 100 + result.txMpps = (result.tx) / duration / 1000000 + result.rxMpps = (result.rx) / duration / 1000000 + result.txRate = (result.tx * frameSize * 8) / duration / 1000000000 + result.rxRate = (result.rx * frameSize * 8) / duration / 1000000000 + + print(getResultString(result)) + return result +end + +function master(args) + local txDev = device.config({port = args.txport, rxQueues = 1, txQueues = 1}) + local rxDev = device.config({port = args.rxport, rxQueues = 1, txQueues = 1}) + local frameSize = args.frameSize + local duration = args.duration + device.waitForLinks() + + local maxLinkRate = txDev:getLinkStatus().speed + local results = {} + + -- Warming up + print ("Output file is: "..args.out) + print ("Start Warm-Up") + runMeasure(txDev, rxDev, frameSize, 1, maxLinkRate) + runMeasure(txDev, rxDev, frameSize, 1, 1000) + print ("Stop Warm-Up") + + local sortFunction = function(a,b) + return a.rate < b.rate + end + + local file = io.open(args.out, "w") + io.output(file) + io.write(getHeaderString().."\n") + io.close(file) + + print ("Start Measures") + print(getHeaderString()) + table.insert(results, runMeasure(txDev, rxDev, frameSize, duration, maxLinkRate)) + table.insert(results, runMeasure(txDev, rxDev, frameSize, duration, maxLinkRate/2)) + table.insert(results, runMeasure(txDev, rxDev, frameSize, duration, 100)) + table.sort(results, sortFunction) + + while mg.running() do + local chosen = nil + local chosenMeaning = 0 + for i, r1 in ipairs(results) do + local r2 = results[i+1] + if r2 == nil then + break + end + local rateDiff = r2.rate - r1.rate + local dropRateRatio = (r2.dropRate + 0.00000001)/(r1.dropRate + 0.00000001) + if r2.dropRate < r1.dropRate then + dropRateRatio = (r1.dropRate+ 0.00000001)/(r2.dropRate + 0.00000001) + end + -- Meaning rates next measures + -- The idea is to compute interesting results first and get more picky later. + local meaning = rateDiff / ((args.maxRateInterval * maxLinkRate) / 100 ) + math.log(dropRateRatio / args.targetDropRateRatio)/math.log(2) + if rateDiff > (args.maxRateInterval * maxLinkRate) / 100 + or (dropRateRatio > args.targetDropRateRatio and (rateDiff) > (args.minRateInterval * maxLinkRate) / 100) then + if meaning > chosenMeaning then + chosenMeaning = meaning + chosen = i + end + end + end + if (chosen == nil) then + break + end + local nextRate = (results[chosen+1].rate + results[chosen].rate)/2 + table.insert(results, runMeasure(txDev, rxDev, frameSize, duration, nextRate)) + table.sort(results, sortFunction) + + local file = io.open(args.out, "w") + io.output(file) + io.write(getHeaderString().."\n") + for i, r1 in ipairs(results) do + io.write(getResultString(r1).."\n") + end + io.close(file) + end + + +end + +function loadSlave(queue, frameSize, duration, bar) + bar:wait() + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethSrc = queue, + ethDst = ETH_DST, + ethType = 0x1234 + } + end) + + local bufs = mem:bufArray() + local timer = timer:new(duration) + local total = 0; + while timer:running() do + bufs:alloc(frameSize) + total = total + queue:send(bufs) + end + return total +end + +function counterSlave(queue, frameSize, duration, bar) + local bufs = memory.bufArray() + local total = 0; + bar:wait() + + local timer = timer:new(duration + 1) + while timer:running() do + total = total + queue:tryRecv(bufs, 1000) + bufs:freeAll() + end + return total +end + diff --git a/vhost-test/vhost.sh b/vhost-test/vhost.sh index 552bdc5..fc3dce5 100755 --- a/vhost-test/vhost.sh +++ b/vhost-test/vhost.sh @@ -21,6 +21,7 @@ VPP_BUILD="xxx" VPP_INSTALL="xxx" VPP="xxx" DPDK_BIND="xxx" +VPPCTL="xxx" CORES_VM_LIST="xxx" CORES_VM_N="xxx" @@ -102,7 +103,7 @@ function load_config() { #Validate config validate_parameter VPP_DIR VPP_IF0_PCI VPP_IF0_MAC VPP_IF1_PCI VPP_IF1_MAC VPP_IF0_NAME VPP_IF1_NAME \ - QEMU VM_ROOT VM_INITRD VM_VMLINUZ VM_VNCPORT VM_USERNAME + QEMU VM_ROOT VM_INITRD VM_VMLINUZ VM_VNCPORT VM_USERNAME VPP_VHOST_MODE VIRTIO_QSZ validate_directory VPP_DIR VM_ROOT validate_file VM_INITRD VM_VMLINUZ validate_exec QEMU @@ -121,6 +122,10 @@ function load_config() { echo "Invalid VPP_BUILD parameter '$VPP_BUILD'" && exit 1 fi + if [ "$VPP_VHOST_MODE" != "client" -a "$VPP_VHOST_MODE" != "server" ]; then + echo "Invalid VPP_VHOST_MODE (must be client or server)" && exit 1 + fi + if [ "$QUEUES" != "1" -a "$QUEUES" != "2" ]; then echo "QUEUES can only be 1 or 2" exit 7 @@ -128,6 +133,7 @@ function load_config() { VPP="$VPP_INSTALL/vpp/bin/vpp" DPDK_BIND="$(ls $VPP_BUILD/dpdk/dpdk-*/tools/dpdk-devbind.py | head -n 1)" + VPPCTL="sudo env PATH=$PATH:${VPP_INSTALL}/../install-vpp_debug-native/vpp-api-test/bin/ vppctl" validate_exec VPP DPDK_BIND @@ -215,7 +221,7 @@ EOF cat > "$VMDIR/etc/rc.local" << EOF #!/bin/sh mkdir -p /var/log/startup/ -for exe in \`ls /etc/startup.d\`; do +for exe in \$(ls /etc/startup.d); do echo -n "Startup script \$exe " ( (nohup /etc/startup.d/\$exe > /var/log/startup/\$exe 2>&1 &) && echo "[OK]") || echo "[Failed]" done @@ -267,6 +273,16 @@ EOF fi LAST_VM_CPU="$(expr $CORES_VM_N - 1)" + VM_VH_SERV_PARAM="" + if [ "$VPP_VHOST_MODE" = "client" ]; then + VM_VH_SERV_PARAM=",server" + fi + + QSZ="" + if [ "$VIRTIO_QSZ" != "256" ]; then + QSZ=",rx_virtqueue_sz=$VIRTIO_QSZ,tx_virtqueue_sz=$VIRTIO_QSZ" + fi + cat << EOF > "$TMP_DIR/vm.conf" -enable-kvm -machine pc -initrd $VM_INITRD -kernel $VM_VMLINUZ -vnc 127.0.0.1:1 -m 4G -append 'root=ro ro rootfstype=9p rootflags=trans=virtio nohz_full=1-$LAST_VM_CPU isolcpus=1-$LAST_VM_CPU rcu_nocbs=1-$LAST_VM_CPU selinux=0 audit=0 net.ifnames=0 biosdevname=0' @@ -277,15 +293,135 @@ EOF -device virtio-9p-pci,id=dev_fs,fsdev=fsdev_id,mount_tag=ro -daemonize -pidfile $TMP_DIR/qemu.pid --chardev socket,id=chr0,path=$TMP_DIR/sock0,server +-chardev socket,id=chr0,path=$TMP_DIR/sock0${VM_VH_SERV_PARAM} -netdev type=vhost-user,id=thrnet0,chardev=chr0,queues=$QUEUES --device virtio-net-pci,netdev=thrnet0,mac=de:ad:be:ef:01:00,bus=pci.0,addr=7.0${MQ} +-device virtio-net-pci,netdev=thrnet0,mac=de:ad:be:ef:01:00,bus=pci.0,addr=7.0,mrg_rxbuf=on,indirect_desc=on${MQ}${QSZ} -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem --chardev socket,id=chr1,path=$TMP_DIR/sock1,server +-chardev socket,id=chr1,path=$TMP_DIR/sock1${VM_VH_SERV_PARAM} -netdev type=vhost-user,id=thrnet1,chardev=chr1,queues=$QUEUES --device virtio-net-pci,netdev=thrnet1,mac=de:ad:be:ef:01:01,bus=pci.0,addr=8.0${MQ} +-device virtio-net-pci,netdev=thrnet1,mac=de:ad:be:ef:01:01,bus=pci.0,addr=8.0,mrg_rxbuf=on,indirect_desc=on${MQ}${QSZ} +EOF +} + +function get_vhost_thread_config() { + VH_IF0="$1" + VH_IF1="$2" + + if [ "$VH_IF0" = "" -o "$VH_IF1" = "" ]; then + echo "Missing get_vhost_thread_config interface argument" + exit 1 + fi + + DEL="" + if [ "$3" = "del" ]; then + DEL="del" + elif [ "$3" != "" ]; then + echo "Invalid get_vhost_thread_config argument" + exit 1 + fi + + #VHOST queue pining + if [ "$QUEUES" = "1" ]; then + if [ "$CORES_VPP_N" = "0" ]; then + echo -n "" + elif [ "$CORES_VPP_N" = "1" ]; then + echo "vhost thread $VH_IF1 1 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + elif [ "$CORES_VPP_N" -lt "4" ]; then + echo "vhost thread $VH_IF1 2 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + else + echo "vhost thread $VH_IF1 3 ${DEL}" + echo "vhost thread $VH_IF0 4 ${DEL}" + fi + elif [ "$QUEUES" = "2" ]; then + if [ "$CORES_VPP_N" = "0" ]; then + echo -n "" + elif [ "$CORES_VPP_N" = "1" ]; then + echo "vhost thread $VH_IF1 1 ${DEL}" + echo "vhost thread $VH_IF1 1 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + elif [ "$CORES_VPP_N" -lt "4" ]; then + echo "vhost thread $VH_IF1 2 ${DEL}" + echo "vhost thread $VH_IF1 2 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + else + echo "vhost thread $VH_IF1 3 ${DEL}" + echo "vhost thread $VH_IF1 4 ${DEL}" + echo "vhost thread $VH_IF0 1 ${DEL}" + echo "vhost thread $VH_IF0 2 ${DEL}" + fi + fi +} + +function disconnect_vhost() { + VH_INST=$1 + + if [ ! -f "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" ]; then + echo "No configured vhost devices $VH_INST" + exit 1 + fi + + echo "------- Disconnect vhost --------" + cat "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" + echo "-------------------------------" + + $VPPCTL exec "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" + rm "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" +} + +function connect_vhost() { + VH_INST=$1 + + if [ -f "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" ]; then + echo "Vhost devices $VH_INST already configured" + exit 1 + fi + + VH_SERV_PARAM="" + if [ "$VPP_VHOST_MODE" = "server" ]; then + VH_SERV_PARAM="server" + fi + echo "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1" + VH_IF0=$($VPPCTL "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1") + VH_IF1=$($VPPCTL "create vhost-user socket $TMP_DIR/sock1$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b2") + + if [ "$VH_INST" = "" ]; then + cat << EOF > "$TMP_DIR/vpp-vhost-connect$VH_INST.conf" +set interface l2 xconnect $VH_IF0 $VPP_IF0_NAME +set interface l2 xconnect $VH_IF1 $VPP_IF1_NAME +set interface l2 xconnect $VPP_IF0_NAME $VH_IF0 +set interface l2 xconnect $VPP_IF1_NAME $VH_IF1 +set interface state $VH_IF0 up +set interface state $VH_IF1 up +EOF + else + cat << EOF > "$TMP_DIR/vpp-vhost-connect$VH_INST.conf" +set interface state $VH_IF0 up +set interface state $VH_IF1 up EOF + fi + + cat << EOF > "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" +delete vhost-user $VH_IF0 +delete vhost-user $VH_IF1 +EOF + + if [ "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then + get_vhost_thread_config $VH_IF0 $VH_IF1 >> "$TMP_DIR/vpp-vhost-connect$VH_INST.conf" + #get_vhost_thread_config $VH_IF0 $VH_IF1 del >> "$TMP_DIR/vpp-vhost-disconnect$VH_INST.conf" + fi + + echo "------- Connect vhost --------" + echo "create vhost-user socket $TMP_DIR/sock0$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b1" + echo "create vhost-user socket $TMP_DIR/sock1$VH_INST $VH_SERV_PARAM hwaddr aa:aa:aa:aa:bb:b2" + cat "$TMP_DIR/vpp-vhost-connect$VH_INST.conf" + echo "-------------------------------" + + $VPPCTL exec "$TMP_DIR/vpp-vhost-connect$VH_INST.conf" } function prepare_vpp() { @@ -320,65 +456,9 @@ dpdk { dev $VPP_IF0_PCI { $VPP_DEV0 } dev $VPP_IF1_PCI { $VPP_DEV1 } } EOF cat << EOF > "$TMP_DIR/vpp.conf" -create vhost-user socket $TMP_DIR/sock0 hwaddr aa:aa:aa:aa:bb:b1 -create vhost-user socket $TMP_DIR/sock1 hwaddr aa:aa:aa:aa:bb:b2 -set interface l2 xconnect VirtualEthernet0/0/0 $VPP_IF0_NAME -set interface l2 xconnect VirtualEthernet0/0/1 $VPP_IF1_NAME -set interface l2 xconnect $VPP_IF0_NAME VirtualEthernet0/0/0 -set interface l2 xconnect $VPP_IF1_NAME VirtualEthernet0/0/1 -set interface state VirtualEthernet0/0/0 up -set interface state VirtualEthernet0/0/1 up set interface state $VPP_IF1_NAME up set interface state $VPP_IF0_NAME up EOF - - #VHOST queue pining - if [ "$QUEUES" = "1" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then - if [ "$CORES_VPP_N" = "0" ]; then - echo -n "" - elif [ "$CORES_VPP_N" = "1" ]; then - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 1 -vhost thread VirtualEthernet0/0/0 1 -EOF - elif [ "$CORES_VPP_N" -lt "4" ]; then - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 2 -vhost thread VirtualEthernet0/0/0 1 -EOF - else - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 3 -vhost thread VirtualEthernet0/0/0 4 -EOF - fi - elif [ "$QUEUES" = "2" -a "$USE_DEFAULT_VHOST_PLACEMENT" != "1" ]; then - if [ "$CORES_VPP_N" = "0" ]; then - echo -n "" - elif [ "$CORES_VPP_N" = "1" ]; then - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 1 -vhost thread VirtualEthernet0/0/1 1 -vhost thread VirtualEthernet0/0/0 1 -vhost thread VirtualEthernet0/0/0 1 -EOF - elif [ "$CORES_VPP_N" -lt "4" ]; then - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 2 -vhost thread VirtualEthernet0/0/1 2 -vhost thread VirtualEthernet0/0/0 1 -vhost thread VirtualEthernet0/0/0 1 -EOF - else - cat << EOF >> "$TMP_DIR/vpp.conf" -vhost thread VirtualEthernet0/0/1 3 -vhost thread VirtualEthernet0/0/1 4 -vhost thread VirtualEthernet0/0/0 1 -vhost thread VirtualEthernet0/0/0 2 -EOF - fi - fi - } function prepare() { @@ -389,6 +469,11 @@ function prepare() { } function start_vpp() { + if [ -f "$TMP_DIR/vpp-running" ]; then + echo "VPP is already running" + return 1 + fi + GDB="" if [ "$VPP_GDB" = "1" ]; then [ -e "$TMP_DIR/vpp.sh.gdbinit" ] && sudo rm "$TMP_DIR/vpp.sh.gdbinit" @@ -400,7 +485,7 @@ run EOF GDB="gdb -x $TMP_DIR/vpp.sh.gdbinit --args " fi - + echo "------- Starting VPP --------" echo " Screen $VPPSCREEN (sudo screen -r $VPPSCREEN)" echo " Command-line Conf:" @@ -408,16 +493,30 @@ EOF echo " CLI Conf:" cat $TMP_DIR/vpp.conf echo "-----------------------------" - + sudo screen -d -m -S "$VPPSCREEN" $GDB $VPP_DIR/build-root/install-vpp-native/vpp/bin/vpp -c $TMP_DIR/vpp.cmdline + touch "$TMP_DIR/vpp-running" +} + +function stop_vpp() { + set +e + sudo screen -S "$VPPSCREEN" -X quit && echo "Stopping VPP" + [ -f "$TMP_DIR/vpp-running" ] && rm "$TMP_DIR/vpp-running" } function start_vm() { + if [ -f "$TMP_DIR/qemu.pid" ]; then + echo "VM already running" + return 1 + fi + echo "------- Starting VM --------" echo " VM conf:" cat $TMP_DIR/vm.conf echo "----------------------------" + vmdir_mount + #Eval is used so that ' characters are not ignored eval sudo chrt -rr 1 taskset -c $CORES_VM $QEMU $(cat $TMP_DIR/vm.conf) @@ -430,6 +529,17 @@ function start_vm() { sudo ip link set $VMTAP up } +function stop_vm() { + set +e + + [ -f "$TMP_DIR/qemu.pid" ] && echo "Stopping VM ($(sudo cat $TMP_DIR/qemu.pid))" && sudo kill "$(sudo cat $TMP_DIR/qemu.pid)" && sudo rm $TMP_DIR/qemu.pid + + sudo ip link set $BRNAME down + sudo brctl delbr $BRNAME + + vmdir_umount +} + function start() { if [ -f "$TMP_DIR/.started" ]; then echo "$TMP_DIR/.started exists" @@ -439,17 +549,14 @@ function start() { fi banner - prepare touch "$TMP_DIR/.started" - echo "0" | sudo tee /proc/sys/kernel/watchdog_cpumask start_vpp - - vmdir_mount - + sleep 10 + connect_vhost start_vm } @@ -465,25 +572,40 @@ function pin_vm() { sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p idx=$(expr $idx + 1) done + + #pin non-running processes to other cores + if [ "${CORES_VM_ARRAY[$idx]}" != "" ]; then + PIDS=$(ps -eLf | grep qemu-system-x86_64 | awk '$5 < 20 { print $4; }') + for p in $PIDS; do + echo "VM lazy process $p on core ${CORES_VM_ARRAY[$idx]}" + sudo taskset -pc ${CORES_VM_ARRAY[$idx]} $p || echo err + done + fi } function pin_vpp() { - + for i in $(ls /proc/irq/ | grep [0-9]); do echo 1 | sudo tee /proc/irq/$i/smp_affinity > /dev/null || true ; done PIDS=$(ps -eLf | grep /bin/vpp | awk '$5 > 50 { print $4; }') + skip_first=1 idx=0 for p in $PIDS; do if [ "${CORES_VPP_ARRAY[$idx]}" = "" ]; then echo "Too many working threads in VPP" return 1 fi - echo "VPP PID $p on core ${CORES_VPP_ARRAY[$idx]}" - sudo taskset -pc ${CORES_VPP_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p - idx=$(expr $idx + 1) + if [ "$skip_first" = "1" ]; then + echo "Skipping $p" + skip_first=0 + else + echo "VPP PID $p on core ${CORES_VPP_ARRAY[$idx]}" + sudo taskset -pc ${CORES_VPP_ARRAY[$idx]} $p && sudo chrt -r -p 1 $p + idx=$(expr $idx + 1) + fi done } @@ -494,19 +616,21 @@ function pin() { function stop() { set +e - - [ -f "$TMP_DIR/qemu.pid" ] && echo "Stopping VM ($(sudo cat $TMP_DIR/qemu.pid))" && sudo kill "$(sudo cat $TMP_DIR/qemu.pid)" && sudo rm $TMP_DIR/qemu.pid - - vmdir_umount - - sudo ip link set $BRNAME down - sudo brctl delbr $BRNAME - - sudo screen -S "$VPPSCREEN" -X quit && echo "Stopping VPP" - + stop_vm + stop_vpp [ -f "$TMP_DIR/.started" ] && rm "$TMP_DIR/.started" } +function cmd_stop_vm() { + load_config + stop_vm +} + +function cmd_start_vm() { + load_config + start_vm +} + function cmd_openvnc() { load_config echo Please VNC to 5900 to connect to this VM console @@ -560,6 +684,16 @@ function cmd_config() { load_config } +function cmd_disconnect() { + load_config + disconnect_vhost $@ +} + +function cmd_connect() { + load_config + connect_vhost $@ +} + [ "$1" = "" ] && echo "Missing arguments" && usage && exit 1 CMD="$1" shift |