1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
|
# Copyright (c) 2019 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -exuo pipefail
# This library defines functions used by multiple entry scripts.
# Keep functions ordered alphabetically, please.
function activate_wrapper () {
# Acts as wrapper for activate docker topology.
#
# Variables read:
# - ${1} - Node multiplicity of desired testbed.
# - ${2} - Node flavor string, usually describing the processor.
# - ${3} - CSIT-SUT-DCR image name and version.
set -exuo pipefail
enter_mutex || die
get_available_interfaces "${1}" "${2}" || die
start_topology_containers "${3}" || die
bind_interfaces_to_containers || die
set_env_variables || die
print_env_variables || die
exit_mutex || die
}
function bind_interfaces_to_containers () {
# Bind linux network interface to container and create symlink for PCI
# address in container.
#
# Variables read:
# - DCR_UUIDS - Docker Container UUIDs.
# - DCR_CPIDS - Docker Container PIDs (namespaces).
# - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
# - PCI_ADDR - PCI address of network device.
# - TG_NETDEVS - List of network devices allocated to TG container.
# Variables set:
# - NETDEV - Linux network interface.
set -exuo pipefail
for NETDEV in "${TG_NETDEVS[@]}"; do
get_pci_addr || die
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
}
docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
done
for NETDEV in "${DUT1_NETDEVS[@]}"; do
get_pci_addr || die
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
}
docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
done
}
function bind_interfaces_to_driver () {
# Bind network interface specified by parameter to driver specified by
# parameter.
#
# Variables read:
# - ADDR - PCI address of network interface.
# - DRIVER - Kernel driver.
pci_path="/sys/bus/pci/devices/${ADDR}"
drv_path="/sys/bus/pci/drivers/${DRIVER}"
vd="$(cat ${pci_path}/vendor ${pci_path}/device)" || {
die "Failed to retrieve interface details!"
}
set +e
echo ${vd} | sudo tee ${drv_path}/new_id
set -e
echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
die "Failed to unbind interface ${ADDR}!"
}
echo ${ADDR} | sudo tee ${drv_path}/bind || {
die "Failed to bind interface ${ADDR}!"
}
}
function clean_environment () {
# Cleanup environment by removing topology containers and shared volumes
# and binding interfaces back to original driver.
#
# Variables read:
# - DCR_UUIDS - Docker Container UUIDs.
# - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
# - TG_PCIDEVS - List of PCI addresses of devices of TG container.
# Variables set:
# - ADDR - PCI address of network interface.
# - DRIVER - Kernel driver.
set -exuo pipefail
# Kill docker containers.
docker rm --force "${DCR_UUIDS[@]}" || die "Cleanup containers failed!"
# Remove DUT1 /tmp volume
docker volume rm "${DCR_VOLUMES[dut1]}" || {
die "Failed to remove DUT1 /tmp volume!"
}
# Rebind interfaces back to kernel drivers.
for ADDR in ${TG_PCIDEVS[@]}; do
DRIVER="${TG_DRIVERS[0]}"
bind_interfaces_to_driver || die
done
for ADDR in ${DUT1_PCIDEVS[@]}; do
DRIVER="${DUT1_DRIVERS[0]}"
bind_interfaces_to_driver || die
done
}
function clean_environment_on_exit () {
# Cleanup environment by removing topology containers and binding
# interfaces back to original driver only if exit code is not 0.
# This function acts as workaround as 'set -eu' does not trigger ERR trap.
if [ $? -ne 0 ]; then
clean_environment || die
fi
}
function deactivate_wrapper () {
# Acts as wrapper for deactivate docker topology.
#
# Variables read:
# - ${@} - CSIT environment variables.
set -exuo pipefail
enter_mutex || die
read_env_variables "${@}" || die
clean_environment || die
exit_mutex || die
}
function die () {
# Print the message to standard error end exit with error code specified
# by the second argument.
#
# Hardcoded values:
# - The default error message.
# Arguments:
# - ${1} - The whole error message, be sure to quote. Optional
# - ${2} - the code to exit with, default: 1.
set -x
set +eu
warn "${1:-Unspecified run-time error occurred!}"
exit "${2:-1}"
}
function enter_mutex () {
# Enter mutual exclusion for protecting execution from starvation and
# deadlock.
set -exuo pipefail
mutex_timeout=3600
mutex_file="/tmp/mutex_file"
# Create mutex.
exec {lock_fd}>${mutex_file} || {
die "Mutex enter failed!"
}
flock --timeout "${mutex_timeout}" "${lock_fd}" || {
die "Calling flock() failed!"
}
# ----------------------
# Enter mutex succeeded.
warn "Mutex enter succeeded for PID $$."
}
function exit_mutex () {
# Exit mutual exclusion.
set -exuo pipefail
# ---------------------
# Remove mutex so we are not blocking others anymore.
flock -u "${lock_fd}" || {
die "Mutex destroy failed!"
}
warn "Mutex leave succeeded for PID $$."
}
function get_available_interfaces () {
# Find and get available Virtual functions.
#
# Arguments:
# - ${1} - Node flavor string, usually describing the processor and node
# multiplicity of desired testbed, separated by underscore.
# Variables set:
# - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
# - DUT1_PCIDEVS - List of PCI addresses allocated to DUT1 container.
# - DUT1_NETMACS - List of MAC addresses allocated to DUT1 container.
# - DUT1_DRIVERS - List of interface drivers to DUT1 container.
# - TG_NETDEVS - List of network devices allocated to TG container.
# - TG_PCIDEVS - List of PCI addresses allocated to TG container.
# - TG_NETMACS - List of MAC addresses allocated to TG container.
# - TG_DRIVERS - List of interface drivers to TG container.
set -exuo pipefail
# Following code is specifing VFs ID based on nodeness and flavor.
# As there is great variability in hardware configuration outside LF,
# from bootstrap architecure point of view these are considered as flavors.
# Anyone can override flavor for its own machine and add condition here.
# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info.
case_text="${1}_${2}"
case "${case_text}" in
"1n_skx")
# Add Intel Corporation XL710/X710 Virtual Function to the
# whitelist.
pci_id="0x154c"
tg_netdev=(enp24)
dut1_netdev=(enp59)
;;
"1n_vbox")
# Add Intel Corporation 82545EM Gigabit Ethernet Controller to the
# whitelist.
pci_id="0x100f"
tg_netdev=(eth1 eth2)
dut1_netdev=(eth3 eth4)
;;
*)
die "Unknown specification: ${case_text}!"
esac
net_path="/sys/bus/pci/devices/*/net/*"
# TG side of connections.
TG_NETDEVS=()
TG_PCIDEVS=()
TG_NETMACS=()
TG_DRIVERS=()
# DUT1 side of connections.
DUT1_NETDEVS=()
DUT1_PCIDEVS=()
DUT1_NETMACS=()
DUT1_DRIVERS=()
# Following code is filtering available VFs represented by network device
# name. Only allowed VFs PCI IDs are used.
for netdev in \
$(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';');
do
if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then
# We will filter to TG/DUT1 side of connection (this can be in
# future overriden by more advanced conditions for mapping).
for sub in ${tg_netdev[@]}; do
if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
tg_side+=(${netdev})
fi
done
for sub in ${dut1_netdev[@]}; do
if [[ "${netdev#*$sub}" != "${netdev}" ]]; then
dut1_side+=(${netdev})
fi
done
fi
done
for netdev in "${tg_side[@]::2}"; do
TG_NETDEVS+=(${netdev})
done
for netdev in "${dut1_side[@]::2}"; do
DUT1_NETDEVS+=(${netdev})
done
for NETDEV in "${TG_NETDEVS[@]}"; do
get_pci_addr
get_mac_addr
get_krn_driver
TG_PCIDEVS+=(${PCI_ADDR})
TG_NETMACS+=(${MAC_ADDR})
TG_DRIVERS+=(${KRN_DRIVER})
done
for NETDEV in "${DUT1_NETDEVS[@]}"; do
get_pci_addr
get_mac_addr
get_krn_driver
DUT1_PCIDEVS+=(${PCI_ADDR})
DUT1_NETMACS+=(${MAC_ADDR})
DUT1_DRIVERS+=(${KRN_DRIVER})
done
# We need at least two interfaces for TG/DUT1 for building topology.
if [ "${#TG_NETDEVS[@]}" -ne 2 ] || [ "${#DUT1_NETDEVS[@]}" -ne 2 ]; then
die "Not enough linux network interfaces found!"
fi
if [ "${#TG_PCIDEVS[@]}" -ne 2 ] || [ "${#DUT1_PCIDEVS[@]}" -ne 2 ]; then
die "Not enough pci interfaces found!"
fi
}
function get_krn_driver () {
# Get kernel driver from linux network device name.
#
# Variables read:
# - PCI_ADDR - PCI address of network device.
# Variables set:
# - KRN_DRIVER - Kernel driver of network device.
set -exuo pipefail
pci_path="/sys/bus/pci/devices/${PCI_ADDR}"
KRN_DRIVER="$(basename $(readlink -f ${pci_path}/driver))" || {
die "Failed to get kernel driver of PCI interface!"
}
}
function get_mac_addr () {
# Get MAC address from linux network device name.
#
# Variables read:
# - NETDEV - Linux network device name.
# Variables set:
# - MAC_ADDR - MAC address of network device.
set -exuo pipefail
if [ -d /sys/class/net/${NETDEV}/device ]; then
MAC_ADDR="$(</sys/class/net/${NETDEV}/address)" || {
die "Failed to get MAC address of linux network interface!"
}
fi
}
function get_pci_addr () {
# Get PCI address in <domain>:<bus:<device>.<func> format from linux network
# device name.
#
# Variables read:
# - NETDEV - Linux network device name.
# Variables set:
# - PCI_ADDR - PCI address of network device.
set -exuo pipefail
if [ -d /sys/class/net/${NETDEV}/device ]; then
PCI_ADDR=$(basename $(readlink /sys/class/net/${NETDEV}/device)) || {
die "Failed to get PCI address of linux network interface!"
}
fi
if [ ! -d /sys/bus/pci/devices/${PCI_ADDR} ]; then
die "PCI device ${NETDEV} doesn't exist!"
fi
}
function installed () {
set -exuo pipefail
# Check if the given utility is installed. Fail if not installed.
#
# Arguments:
# - ${1} - Utility to check.
# Returns:
# - 0 - If command is installed.
# - 1 - If command is not installed.
command -v "${1}"
}
function print_env_variables () {
# Get environment variables prefixed by CSIT_.
set -exuo pipefail
env | grep CSIT_
}
function read_env_variables () {
# Read environment variables from parameters.
#
# Arguments:
# - ${@} - Variables passed as an argument.
set -exuo pipefail
for param in "$@"; do
export "${param}"
done
declare -gA DCR_UUIDS
declare -gA DCR_VOLUMES
DCR_UUIDS+=([tg]="${CSIT_TG_UUID}")
DCR_UUIDS+=([dut1]="${CSIT_DUT1_UUID}")
DCR_VOLUMES+=([dut1]="${CSIT_DUT1_VOL}")
TG_PCIDEVS=("${CSIT_TG_INTERFACES_PORT1_PCI}")
TG_DRIVERS=("${CSIT_TG_INTERFACES_PORT1_DRV}")
TG_PCIDEVS+=("${CSIT_TG_INTERFACES_PORT2_PCI}")
TG_DRIVERS+=("${CSIT_TG_INTERFACES_PORT2_DRV}")
DUT1_PCIDEVS=("${CSIT_DUT1_INTERFACES_PORT1_PCI}")
DUT1_DRIVERS=("${CSIT_DUT1_INTERFACES_PORT1_DRV}")
DUT1_PCIDEVS+=("${CSIT_DUT1_INTERFACES_PORT2_PCI}")
DUT1_DRIVERS+=("${CSIT_DUT1_INTERFACES_PORT2_DRV}")
}
function set_env_variables () {
# Set environment variables.
#
# Variables read:
# - DCR_UUIDS - Docker Container UUIDs.
# - DCR_PORTS - Docker Container's SSH ports.
# - DUT1_NETMACS - List of network devices MAC addresses of DUT1 container.
# - DUT1_PCIDEVS - List of PCI addresses of devices of DUT1 container.
# - DUT1_DRIVERS - List of interface drivers to DUT1 container.
# - TG_NETMACS - List of network devices MAC addresses of TG container.
# - TG_PCIDEVS - List of PCI addresses of devices of TG container.
# - TG_DRIVERS - List of interface drivers to TG container.
set -exuo pipefail
set -a
CSIT_TG_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
die "Reading hostname IP address failed!"
}
CSIT_TG_PORT="${DCR_PORTS[tg]#*:}"
CSIT_TG_UUID="${DCR_UUIDS[tg]}"
CSIT_TG_ARCH="$(uname -i)" || {
die "Reading machine architecture failed!"
}
CSIT_DUT1_HOST="$(hostname --all-ip-addresses | awk '{print $1}')" || {
die "Reading hostname IP address failed!"
}
CSIT_DUT1_PORT="${DCR_PORTS[dut1]#*:}"
CSIT_DUT1_UUID="${DCR_UUIDS[dut1]}"
CSIT_DUT1_ARCH="$(uname -i)" || {
die "Reading machine architecture failed!"
}
CSIT_DUT1_VOL="${DCR_VOLUMES[dut1]}"
CSIT_TG_INTERFACES_PORT1_MAC="${TG_NETMACS[0]}"
CSIT_TG_INTERFACES_PORT1_PCI="${TG_PCIDEVS[0]}"
CSIT_TG_INTERFACES_PORT1_DRV="${TG_DRIVERS[0]}"
CSIT_TG_INTERFACES_PORT2_MAC="${TG_NETMACS[1]}"
CSIT_TG_INTERFACES_PORT2_PCI="${TG_PCIDEVS[1]}"
CSIT_TG_INTERFACES_PORT2_DRV="${TG_DRIVERS[1]}"
CSIT_DUT1_INTERFACES_PORT1_MAC="${DUT1_NETMACS[0]}"
CSIT_DUT1_INTERFACES_PORT1_PCI="${DUT1_PCIDEVS[0]}"
CSIT_DUT1_INTERFACES_PORT1_DRV="${DUT1_DRIVERS[0]}"
CSIT_DUT1_INTERFACES_PORT2_MAC="${DUT1_NETMACS[1]}"
CSIT_DUT1_INTERFACES_PORT2_PCI="${DUT1_PCIDEVS[1]}"
CSIT_DUT1_INTERFACES_PORT2_DRV="${DUT1_DRIVERS[1]}"
set +a
}
function start_topology_containers () {
# Starts csit-sut-dcr docker containers for TG/DUT1.
#
# Variables read:
# - CSIT_DIR - Path to existing root of local CSIT git repository.
# Variables set:
# - DCR_UUIDS - Docker Container UUIDs.
# - DCR_PORTS - Docker Container SSH TCP ports.
# - DCR_CPIDS - Docker Container PIDs (namespaces).
set -exuo pipefail
if ! installed docker; then
die "Docker not present. Please install before continue!"
fi
# If the IMAGE is not already loaded then docker run will pull the IMAGE,
# and all image dependencies, before it starts the container.
dcr_image="${1}"
# Run the container in the background and print the new container ID.
dcr_stc_params="--detach=true "
# Give extended privileges to this container. A "privileged" container is
# given access to all devices and able to run nested containers.
dcr_stc_params+="--privileged "
# Publish all exposed ports to random ports on the host interfaces.
dcr_stc_params+="--publish-all "
# Automatically remove the container when it exits.
dcr_stc_params+="--rm "
# Size of /dev/shm.
dcr_stc_params+="--shm-size 512M "
# Override access to PCI bus by attaching a filesystem mount to the
# container.
dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
# Mount vfio to be able to bind to see binded interfaces. We cannot use
# --device=/dev/vfio as this does not see newly binded interfaces.
dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
# Mount nested_vm image to be able to run VM tests.
dcr_stc_params+="--volume /var/lib/vm/vhost-nested.img:/var/lib/vm/vhost-nested.img "
# Mount docker.sock to be able to use docker deamon of the host.
dcr_stc_params+="--volume /var/run/docker.sock:/var/run/docker.sock "
# Docker Container UUIDs.
declare -gA DCR_UUIDS
# Docker Container SSH TCP ports.
declare -gA DCR_PORTS
# Docker Container PIDs (namespaces).
declare -gA DCR_CPIDS
# Docker Container volumes with no relationship to the host.
declare -gA DCR_VOLUMES
# Create DUT1 /tmp volume to be able to install VPP in "nested" container.
params=(--name DUT1_VOL_$(uuidgen))
DCR_VOLUMES+=([dut1]="$(docker volume create "${params[@]}")") || {
die "Failed to create DUT1 /tmp volume!"
}
# Mount DUT1_VOL as /tmp directory on DUT1 container
dcr_stc_params_dut1="--volume ${DCR_VOLUMES[dut1]}:/tmp "
# Run TG and DUT1. As initial version we do support only 2-node.
params=(${dcr_stc_params} --name csit-tg-$(uuidgen) ${dcr_image})
DCR_UUIDS+=([tg]="$(docker run "${params[@]}")") || {
die "Failed to start TG docker container!"
}
params=(${dcr_stc_params} ${dcr_stc_params_dut1}
--name csit-dut1-$(uuidgen) ${dcr_image})
DCR_UUIDS+=([dut1]="$(docker run "${params[@]}")") || {
die "Failed to start DUT1 docker container!"
}
trap 'clean_environment_on_exit' EXIT || {
die "Trap attempt failed, please cleanup manually. Aborting!"
}
# Get Containers TCP ports.
params=(${DCR_UUIDS[tg]})
DCR_PORTS+=([tg]="$(docker port "${params[@]}")") || {
die "Failed to get port of TG docker container!"
}
params=(${DCR_UUIDS[dut1]})
DCR_PORTS+=([dut1]="$(docker port "${params[@]}")") || {
die "Failed to get port of DUT1 docker container!"
}
# Get Containers PIDs.
params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[tg]})
DCR_CPIDS+=([tg]="$(docker inspect "${params[@]}")") || {
die "Failed to get PID of TG docker container!"
}
params=(--format="{{ .State.Pid }}" ${DCR_UUIDS[dut1]})
DCR_CPIDS+=([dut1]="$(docker inspect "${params[@]}")") || {
die "Failed to get PID of DUT1 docker container!"
}
}
function warn () {
# Print the message to standard error.
#
# Arguments:
# - ${@} - The text of the message.
echo "$@" >&2
}
|