diff options
Diffstat (limited to 'vicn/resource/vpp')
| -rw-r--r-- | vicn/resource/vpp/__init__.py | 0 | ||||
| -rw-r--r-- | vicn/resource/vpp/cicn.py | 138 | ||||
| -rw-r--r-- | vicn/resource/vpp/dpdk_device.py | 35 | ||||
| -rw-r--r-- | vicn/resource/vpp/interface.py | 125 | ||||
| -rw-r--r-- | vicn/resource/vpp/scripts.py | 287 | ||||
| -rw-r--r-- | vicn/resource/vpp/vpp.py | 187 | ||||
| -rw-r--r-- | vicn/resource/vpp/vpp_bridge.py | 130 | ||||
| -rw-r--r-- | vicn/resource/vpp/vpp_commands.py | 41 | ||||
| -rw-r--r-- | vicn/resource/vpp/vpp_host.py | 144 |
9 files changed, 1087 insertions, 0 deletions
diff --git a/vicn/resource/vpp/__init__.py b/vicn/resource/vpp/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/vicn/resource/vpp/__init__.py diff --git a/vicn/resource/vpp/cicn.py b/vicn/resource/vpp/cicn.py new file mode 100644 index 00000000..be523a6c --- /dev/null +++ b/vicn/resource/vpp/cicn.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from netmodel.model.type import Integer, Bool +from vicn.core.attribute import Attribute +from vicn.core.exception import ResourceNotFound +from vicn.core.requirement import Requirement +from vicn.core.resource_mgr import wait_resource_task +from vicn.core.task import async_task, task, BashTask, EmptyTask +from vicn.resource.icn.forwarder import Forwarder +from vicn.resource.node import Node +from vicn.resource.vpp.vpp_commands import CMD_VPP_ENABLE_PLUGIN +from vicn.resource.vpp.vpp_commands import CMD_VPP_CICN_GET +from vicn.resource.vpp.vpp_commands import CMD_VPP_ADD_ICN_FACE +from vicn.resource.vpp.vpp_commands import CMD_VPP_ADD_ICN_ROUTE +from vicn.resource.vpp.vpp_commands import CMD_VPP_CICN_GET_CACHE_SIZE +from vicn.resource.vpp.vpp_commands import CMD_VPP_CICN_SET_CACHE_SIZE + +_ADD_FACE_RETURN_FORMAT = "Face ID: [0-9]+" + +def check_face_id_return_format(data): + prog = re.compile(_ADD_FACE_RETURN_FORMAT) + return prog.match(data) + +def parse_face_id(data): + return data.partition(':')[2] + +class CICNForwarder(Forwarder): + """ + NOTE: Based on the Vagrantfile, we recommend a node with mem=4096, cpu=4 + """ + + node = Attribute(Node, + mandatory=True, + requirements = [Requirement('vpp')], + reverse_name='cicn') + numa_node = Attribute(Integer, + description = 'Numa node on which vpp will run', + default = None) + core = Attribute(Integer, + description = 'Core belonging the numa node on which vpp will run', + default = None) + enable_worker = Attribute(Bool, + description = 'Enable one worker for packet processing', + default = False) + + #__packages__ = ['vpp-plugin-cicn'] + + def __after__(self): + return ['CentralICN'] + + def __get__(self): + def parse(rv): + if rv.return_value > 0 or 'cicn: not enabled' in rv.stdout: + raise ResourceNotFound + return BashTask(self.node, CMD_VPP_CICN_GET, + lock = self.node.vpp.vppctl_lock, parse=parse) + + def __create__(self): + + #self.node.vpp.plugins.append("cicn") + lock = self.node.vpp.vppctl_lock + create_task = BashTask(self.node, CMD_VPP_ENABLE_PLUGIN, + {'plugin' : 'cicn'}, lock = lock) + + face_task = EmptyTask() + route_task = EmptyTask() + + def parse_face(rv, face): + if check_face_id_return_format(rv.stdout): + face.id = parse_face_id(rv.stdout) + return {} + + for face in self.faces: + face_task = face_task > BashTask(self.node, CMD_VPP_ADD_ICN_FACE, + {'face':face}, + parse = (lambda x : parse_face(x, face)), lock = lock) + + if not self.routes: + from vicn.resource.icn.route import Route + for route in self._state.manager.by_type(Route): + if route.node is self.node: + self.routes.append(route) + for route in self.routes: + route_task = route_task > BashTask(self.node, + CMD_VPP_ADD_ICN_ROUTE, {'route' : route}, lock = lock) + + return (wait_resource_task(self.node.vpp) > create_task) > (face_task > route_task) + + # Nothing to do + __delete__ = None + + #-------------------------------------------------------------------------- + # Attributes + #-------------------------------------------------------------------------- + + # Force local update + + _add_faces = None + _remove_faces = None + _get_faces = None + _set_faces = None + + _add_routes = None + _remove_routes = None + _get_routes = None + _set_routes = None + + #-------------------------------------------------------------------------- + # Internal methods + #-------------------------------------------------------------------------- + + def _set_cache_size(self): + return BashTask(self.node, CMD_VPP_CICN_SET_CACHE_SIZE, {'self': self}, + lock = self.node.vpp.vppctl_lock) + + def _get_cache_size(self): + def parse(rv): + return int(rv.stdout) + return BashTask(self.node, CMD_VPP_CICN_GET_CACHE_SIZE, parse=parse, + lock = self.node.vpp.vppctl_lock) diff --git a/vicn/resource/vpp/dpdk_device.py b/vicn/resource/vpp/dpdk_device.py new file mode 100644 index 00000000..69449e48 --- /dev/null +++ b/vicn/resource/vpp/dpdk_device.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from netmodel.model.type import Integer, String +from vicn.core.attribute import Attribute +from vicn.resource.linux.phy_interface import PhyInterface + +class DpdkDevice(PhyInterface): + """ + Resource: DpdkDevice + + A DpdkDevice is a physical net device supported by Dpdk and with parameters + specific to VPP. + """ + numa_node = Attribute(Integer, + description = 'NUMA node on the same PCI bus as the DPDK card') + socket_mem = Attribute(Integer, + description = 'Memory used by the vpp forwarder', + default = 512) + mac_address = Attribute(String) diff --git a/vicn/resource/vpp/interface.py b/vicn/resource/vpp/interface.py new file mode 100644 index 00000000..efe4fe5a --- /dev/null +++ b/vicn/resource/vpp/interface.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from netmodel.model.type import Integer, String, Bool +from vicn.core.resource import Resource +from vicn.core.attribute import Attribute, Multiplicity +from vicn.core.exception import ResourceNotFound +from vicn.core.task import inline_task, BashTask, task +from vicn.core.task import EmptyTask +from vicn.resource.interface import Interface +from vicn.resource.linux.net_device import NonTapBaseNetDevice +from vicn.resource.vpp.vpp import VPP +from vicn.resource.vpp.vpp_commands import CMD_VPP_CREATE_IFACE +from vicn.resource.vpp.vpp_commands import CMD_VPP_SET_IP, CMD_VPP_SET_UP + +class VPPInterface(Resource): + """ + Resource: VPPInterface + + An interface representation in VPP + """ + + vpp = Attribute(VPP, + description = 'Forwarder to which this interface belong to', + mandatory = True, + multiplicity = Multiplicity.ManyToOne, + key = True, + reverse_name = 'interfaces') + parent = Attribute(Interface, description = 'parent', + mandatory = True, reverse_name = 'vppinterface') + ip_address = Attribute(String) + prefix_len = Attribute(Integer, default = 31) + up = Attribute(Bool, description = 'Interface up/down status') + monitored = Attribute(Bool, default = True) + + device_name = Attribute(String) + + #-------------------------------------------------------------------------- + # Resource lifecycle + #-------------------------------------------------------------------------- + + def __after__(self): + """ + We need CentralIP to get the parent interface IP address + """ + return ['CentralIP'] + + @inline_task + def __get__(self): + raise ResourceNotFound + + def __create__(self): + # We must control what is the type of the parent netDevice (currently + # supported only veths, physical nics are coming) + create_task = EmptyTask() + + # We must let the routing algorithm know that the parent interface + # belongs to vpp + self.parent.has_vpp_child = True + + self.ip_address = self.parent.ip_address + self.up = True + + if isinstance(self.parent,NonTapBaseNetDevice): + # Remove ip address in the parent device, it must only be set in + # the vpp interface otherwise vpp and the linux kernel will reply + # to non-icn request (e.g., ARP replies, port ureachable etc) + + self.device_name = 'host-' + self.parent.device_name + create_task = BashTask(self.vpp.node, CMD_VPP_CREATE_IFACE, + {'vpp_interface': self}, + lock = self.vpp.vppctl_lock) + + self.parent.set('ip_address', None) + self.parent.set('offload', False) + self.parent.remote.set('offload', False) + + elif self.parent.get_type() == 'dpdkdevice': + self.device_name = self.parent.device_name + else : + # Currently assume naively that everything else will be a physical + # NIC for VPP + # + # Before initialization, we need to make sure that the parent + # interface is down (vpp will control the nic) + self.device_name = 'host-' + self.parent.device_name + self.parent.set('up', False) + + return create_task + + #-------------------------------------------------------------------------- + # Attributes + #-------------------------------------------------------------------------- + + def _set_ip_address(self): + if self.ip_address: + return BashTask(self.vpp.node, CMD_VPP_SET_IP, {'netdevice': self}, + lock = self.vpp.vppctl_lock) + + def _set_up(self): + return BashTask(self.vpp.node, CMD_VPP_SET_UP, {'netdevice': self}, + lock = self.vpp.vppctl_lock) + + @task + def _get_up(self): + return {'up' : False} + + @task + def _get_ip_address(self): + return {'ip_address' : None} diff --git a/vicn/resource/vpp/scripts.py b/vicn/resource/vpp/scripts.py new file mode 100644 index 00000000..3a3d5e8f --- /dev/null +++ b/vicn/resource/vpp/scripts.py @@ -0,0 +1,287 @@ +FN_APPARMOR_DPDK_SCRIPT='/etc/apparmor.d/lxc/lxc-dpdk' + +TPL_APPARMOR_DPDK_SCRIPT=''' +profile lxc-dpdk flags=(attach_disconnected,mediate_deleted) { + ### Base profile + capability, + dbus, + file, + network, + umount, + + # Allow us to receive signals from anywhere. + signal (receive), + + # Allow us to send signals to ourselves + signal peer=@{profile_name}, + + # Allow other processes to read our /proc entries, futexes, perf tracing and + # kcmp for now (they will need 'read' in the first place). Administrators can + # override with: + # deny ptrace (readby) ... + ptrace (readby), + + # Allow other processes to trace us by default (they will need 'trace' in + # the first place). Administrators can override with: + # deny ptrace (tracedby) ... + ptrace (tracedby), + + # Allow us to ptrace ourselves + ptrace peer=@{profile_name}, + + # ignore DENIED message on / remount + deny mount options=(ro, remount) -> /, + deny mount options=(ro, remount, silent) -> /, + + # allow tmpfs mounts everywhere + mount fstype=tmpfs, + + # allow hugetlbfs mounts everywhere + mount fstype=hugetlbfs, + + # allow mqueue mounts everywhere + mount fstype=mqueue, + + # allow fuse mounts everywhere + mount fstype=fuse, + mount fstype=fuse.*, + + # deny access under /proc/bus to avoid e.g. messing with pci devices directly + deny @{PROC}/bus/** wklx, + + # deny writes in /proc/sys/fs but allow binfmt_misc to be mounted + mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/, + deny @{PROC}/sys/fs/** wklx, + + # allow efivars to be mounted, writing to it will be blocked though + mount fstype=efivarfs -> /sys/firmware/efi/efivars/, + + # block some other dangerous paths + deny @{PROC}/kcore rwklx, + deny @{PROC}/kmem rwklx, + deny @{PROC}/mem rwklx, + deny @{PROC}/sysrq-trigger rwklx, + + # deny writes in /sys except for /sys/fs/cgroup, also allow + # fusectl, securityfs and debugfs to be mounted there (read-only) + mount fstype=fusectl -> /sys/fs/fuse/connections/, + mount fstype=securityfs -> /sys/kernel/security/, + mount fstype=debugfs -> /sys/kernel/debug/, + deny mount fstype=debugfs -> /var/lib/ureadahead/debugfs/, + mount fstype=proc -> /proc/, + mount fstype=sysfs -> /sys/, + mount options=(rw, nosuid, nodev, noexec, remount) -> /sys/, + deny /sys/firmware/efi/efivars/** rwklx, + # note, /sys/kernel/security/** handled below + mount options=(move) /sys/fs/cgroup/cgmanager/ -> /sys/fs/cgroup/cgmanager.lower/, + mount options=(ro, nosuid, nodev, noexec, remount, strictatime) -> /sys/fs/cgroup/, + + mount options=(ro, nosuid, nodev, noexec, remount, strictatime) -> /sys, + + # deny reads from debugfs + deny /sys/kernel/debug/{,**} rwklx, + + # allow paths to be made slave, shared, private or unbindable + # FIXME: This currently doesn't work due to the apparmor parser treating those as allowing all mounts. +# mount options=(rw,make-slave) -> **, +# mount options=(rw,make-rslave) -> **, +# mount options=(rw,make-shared) -> **, +# mount options=(rw,make-rshared) -> **, +# mount options=(rw,make-private) -> **, +# mount options=(rw,make-rprivate) -> **, +# mount options=(rw,make-unbindable) -> **, +# mount options=(rw,make-runbindable) -> **, + + # allow bind-mounts of anything except /proc, /sys and /dev + mount options=(rw,bind) /[^spd]*{,/**}, + mount options=(rw,bind) /d[^e]*{,/**}, + mount options=(rw,bind) /de[^v]*{,/**}, + mount options=(rw,bind) /dev/.[^l]*{,/**}, + mount options=(rw,bind) /dev/.l[^x]*{,/**}, + mount options=(rw,bind) /dev/.lx[^c]*{,/**}, + mount options=(rw,bind) /dev/.lxc?*{,/**}, + mount options=(rw,bind) /dev/[^.]*{,/**}, + mount options=(rw,bind) /dev?*{,/**}, + mount options=(rw,bind) /p[^r]*{,/**}, + mount options=(rw,bind) /pr[^o]*{,/**}, + mount options=(rw,bind) /pro[^c]*{,/**}, + mount options=(rw,bind) /proc?*{,/**}, + mount options=(rw,bind) /s[^y]*{,/**}, + mount options=(rw,bind) /sy[^s]*{,/**}, + mount options=(rw,bind) /sys?*{,/**}, + + # allow moving mounts except for /proc, /sys and /dev + mount options=(rw,move) /[^spd]*{,/**}, + mount options=(rw,move) /d[^e]*{,/**}, + mount options=(rw,move) /de[^v]*{,/**}, + mount options=(rw,move) /dev/.[^l]*{,/**}, + mount options=(rw,move) /dev/.l[^x]*{,/**}, + mount options=(rw,move) /dev/.lx[^c]*{,/**}, + mount options=(rw,move) /dev/.lxc?*{,/**}, + mount options=(rw,move) /dev/[^.]*{,/**}, + mount options=(rw,move) /dev?*{,/**}, + mount options=(rw,move) /p[^r]*{,/**}, + mount options=(rw,move) /pr[^o]*{,/**}, + mount options=(rw,move) /pro[^c]*{,/**}, + mount options=(rw,move) /proc?*{,/**}, + mount options=(rw,move) /s[^y]*{,/**}, + mount options=(rw,move) /sy[^s]*{,/**}, + mount options=(rw,move) /sys?*{,/**}, + + # generated by: lxc-generate-aa-rules.py container-rules.base + deny /proc/sys/[^kn]*{,/**} wklx, + deny /proc/sys/k[^e]*{,/**} wklx, + deny /proc/sys/ke[^r]*{,/**} wklx, + deny /proc/sys/ker[^n]*{,/**} wklx, + deny /proc/sys/kern[^e]*{,/**} wklx, + deny /proc/sys/kerne[^l]*{,/**} wklx, + deny /proc/sys/kernel/[^smhd]*{,/**} wklx, + deny /proc/sys/kernel/d[^o]*{,/**} wklx, + deny /proc/sys/kernel/do[^m]*{,/**} wklx, + deny /proc/sys/kernel/dom[^a]*{,/**} wklx, + deny /proc/sys/kernel/doma[^i]*{,/**} wklx, + deny /proc/sys/kernel/domai[^n]*{,/**} wklx, + deny /proc/sys/kernel/domain[^n]*{,/**} wklx, + deny /proc/sys/kernel/domainn[^a]*{,/**} wklx, + deny /proc/sys/kernel/domainna[^m]*{,/**} wklx, + deny /proc/sys/kernel/domainnam[^e]*{,/**} wklx, + deny /proc/sys/kernel/domainname?*{,/**} wklx, + deny /proc/sys/kernel/h[^o]*{,/**} wklx, + deny /proc/sys/kernel/ho[^s]*{,/**} wklx, + deny /proc/sys/kernel/hos[^t]*{,/**} wklx, + deny /proc/sys/kernel/host[^n]*{,/**} wklx, + deny /proc/sys/kernel/hostn[^a]*{,/**} wklx, + deny /proc/sys/kernel/hostna[^m]*{,/**} wklx, + deny /proc/sys/kernel/hostnam[^e]*{,/**} wklx, + deny /proc/sys/kernel/hostname?*{,/**} wklx, + deny /proc/sys/kernel/m[^s]*{,/**} wklx, + deny /proc/sys/kernel/ms[^g]*{,/**} wklx, + deny /proc/sys/kernel/msg*/** wklx, + deny /proc/sys/kernel/s[^he]*{,/**} wklx, + deny /proc/sys/kernel/se[^m]*{,/**} wklx, + deny /proc/sys/kernel/sem*/** wklx, + deny /proc/sys/kernel/sh[^m]*{,/**} wklx, + deny /proc/sys/kernel/shm*/** wklx, + deny /proc/sys/kernel?*{,/**} wklx, + deny /proc/sys/n[^e]*{,/**} wklx, + deny /proc/sys/ne[^t]*{,/**} wklx, + deny /proc/sys/net?*{,/**} wklx, + deny /sys/[^fdck]*{,/**} wklx, + deny /sys/c[^l]*{,/**} wklx, + deny /sys/cl[^a]*{,/**} wklx, + deny /sys/cla[^s]*{,/**} wklx, + deny /sys/clas[^s]*{,/**} wklx, + deny /sys/class/[^nu]*{,/**} wklx, + deny /sys/class/n[^e]*{,/**} wklx, + deny /sys/class/ne[^t]*{,/**} wklx, + deny /sys/class/net?*{,/**} wklx, + deny /sys/class/u[^i]*{,/**} wklx, + deny /sys/class/ui[^o]*{,/**} wklx, + deny /sys/class?*{,/**} wklx, + deny /sys/d[^e]*{,/**} wklx, + deny /sys/de[^v]*{,/**} wklx, + deny /sys/dev[^i]*{,/**} wklx, + deny /sys/devi[^c]*{,/**} wklx, + deny /sys/devic[^e]*{,/**} wklx, + deny /sys/device[^s]*{,/**} wklx, +# deny /sys/devices/[^vu]*{,/**} wklx, +# deny /sys/devices/v[^i]*{,/**} wklx, +# deny /sys/devices/vi[^r]*{,/**} wklx, +# deny /sys/devices/vir[^t]*{,/**} wklx, +# deny /sys/devices/virt[^u]*{,/**} wklx, +# deny /sys/devices/virtu[^a]*{,/**} wklx, +# deny /sys/devices/virtua[^l]*{,/**} wklx, +# deny /sys/devices/virtual/[^n]*{,/**} wklx, +# deny /sys/devices/virtual/n[^e]*{,/**} wklx, +# deny /sys/devices/virtual/ne[^t]*{,/**} wklx, +# deny /sys/devices/virtual/net?*{,/**} wklx, +# deny /sys/devices/virtual?*{,/**} wklx, +# deny /sys/devices?*{,/**} wklx, + deny /sys/f[^s]*{,/**} wklx, + deny /sys/fs/[^c]*{,/**} wklx, + deny /sys/fs/c[^g]*{,/**} wklx, + deny /sys/fs/cg[^r]*{,/**} wklx, + deny /sys/fs/cgr[^o]*{,/**} wklx, + deny /sys/fs/cgro[^u]*{,/**} wklx, + deny /sys/fs/cgrou[^p]*{,/**} wklx, + deny /sys/fs/cgroup?*{,/**} wklx, + deny /sys/fs?*{,/**} wklx, + + ### Feature: unix + # Allow receive via unix sockets from anywhere + unix (receive), + + # Allow all unix in the container + unix peer=(label=@{profile_name}), + + ### Feature: cgroup namespace + mount fstype=cgroup -> /sys/fs/cgroup/**, + + ### Feature: apparmor stacking + + ### Configuration: apparmor loading disabled in privileged containers + deny /sys/k[^e]*{,/**} rwklx, + deny /sys/ke[^r]*{,/**} rwklx, + deny /sys/ker[^n]*{,/**} rwklx, + deny /sys/kern[^e]*{,/**} rwklx, + deny /sys/kerne[^l]*{,/**} rwklx, + deny /sys/kernel/[^sm]*{,/**} rwklx, + deny /sys/kernel/s[^e]*{,/**} rwklx, + deny /sys/kernel/se[^c]*{,/**} rwklx, + deny /sys/kernel/sec[^u]*{,/**} rwklx, + deny /sys/kernel/secu[^r]*{,/**} rwklx, + deny /sys/kernel/secur[^i]*{,/**} rwklx, + deny /sys/kernel/securi[^t]*{,/**} rwklx, + deny /sys/kernel/securit[^y]*{,/**} rwklx, + deny /sys/kernel/security/[^a]*{,/**} rwklx, + deny /sys/kernel/security/a[^p]*{,/**} rwklx, + deny /sys/kernel/security/ap[^p]*{,/**} rwklx, + deny /sys/kernel/security/app[^a]*{,/**} rwklx, + deny /sys/kernel/security/appa[^r]*{,/**} rwklx, + deny /sys/kernel/security/appar[^m]*{,/**} rwklx, + deny /sys/kernel/security/apparm[^o]*{,/**} rwklx, + deny /sys/kernel/security/apparmo[^r]*{,/**} rwklx, + deny /sys/kernel/security/apparmor?*{,/**} rwklx, + deny /sys/kernel/security?*{,/**} rwklx, + deny /sys/kernel?*{,/**} rwklx, +}''' + +FN_VPP_DPDK_SCRIPT='/etc/vpp/startup.conf' + +TPL_VPP_DPDK_DAEMON_SCRIPT=''' +unix { + nodaemon + log /tmp/vpp.log + full-coredump +} + +api-trace { + on +} + +api-segment { + gid vpp +} + +''' + +TPL_VPP_DPDK_SCRIPT=''' +unix { + log /tmp/vpp.log + full-coredump +} + +api-trace { + on +} + +api-segment { + gid vpp +} + +''' + +APPARMOR_VPP_PROFILE = ''' +lxc.aa_profile = lxc-dpdk +lxc.mount.entry = hugetlbfs dev/hugepages hugetlbfs rw,relatime,create=dir 0 0 +lxc.mount.auto = sys:rw''' diff --git a/vicn/resource/vpp/vpp.py b/vicn/resource/vpp/vpp.py new file mode 100644 index 00000000..f9d10703 --- /dev/null +++ b/vicn/resource/vpp/vpp.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio + +from netmodel.model.type import String, Integer, Bool +from vicn.core.attribute import Attribute, Multiplicity +from vicn.core.exception import ResourceNotFound +from vicn.core.resource import Resource +from vicn.core.task import BashTask, task, inline_task +from vicn.resource.lxd.lxc_container import LxcContainer +from vicn.resource.node import Node +from vicn.resource.linux.file import TextFile +from vicn.resource.vpp.dpdk_device import DpdkDevice +from vicn.resource.vpp.scripts import FN_VPP_DPDK_SCRIPT +from vicn.resource.vpp.scripts import TPL_VPP_DPDK_DAEMON_SCRIPT +from vicn.resource.vpp.vpp_commands import CMD_VPP_DISABLE, CMD_VPP_STOP +from vicn.resource.vpp.vpp_commands import CMD_VPP_START +from vicn.resource.vpp.vpp_commands import CMD_VPP_ENABLE_PLUGIN + +#------------------------------------------------------------------------------ +# VPP forwarder +#------------------------------------------------------------------------------ + +CMD_GET = 'killall -0 vpp_main' +CMD_DISABLE_IP_FORWARD = 'sysctl -w net.ipv4.ip_forward=0' + +class VPP(Resource): + """ + Todo: + - make VPP an application with package install + - vpp should be a service (hence a singleton) for which we override the + start and stop commands + """ + + #__package_names__ = ['vpp', 'vpp-dbg', 'vpp-dpdk-dev'] + + plugins = Attribute(String, + multiplicity = Multiplicity.OneToMany) + node = Attribute(Node, + multiplicity = Multiplicity.OneToOne, + reverse_name = 'vpp') + numa_node = Attribute(Integer, + description = 'Numa node on which vpp will run') + core = Attribute(Integer, + description = 'Core belonging the numa node on which vpp will run') + enable_worker = Attribute(Bool, + description = 'Enable one worker for packet processing', + default = False) + + #-------------------------------------------------------------------------- + # Constructor and Accessors + #-------------------------------------------------------------------------- + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.vppctl_lock = asyncio.Lock() + + self.dpdk_setup_file = None + if isinstance(self.node, LxcContainer): + if not 'vpp' in self.node.profiles: + self.node.profiles.append('vpp') + + #-------------------------------------------------------------------------- + # Resource lifecycle + #-------------------------------------------------------------------------- + + def __after__(self): + return ['BaseNetDevice'] + + def __get__(self): + return BashTask(self.node, CMD_GET) + + def __subresources__(self): + self.dpdk_setup_file = TextFile(node = self.node, + filename = FN_VPP_DPDK_SCRIPT, + overwrite = True) + return self.dpdk_setup_file + + def __create__(self): + socket_mem = dict() + numa_mgr = self.node.node_with_kernel.numa_mgr + + for interface in self.node.interfaces: + if isinstance(interface, DpdkDevice): + # Assign as numa node the first numa node specified in a + # physical card (if any). If multiple nics connected to + # different numa nodes are assigned to this vpp memory access + # will be inefficient for the nics sitting in the other numa + # node. + socket_mem[interface.numa_node] = interface.socket_mem + + for iface in self.interfaces: + if isinstance(iface.parent, DpdkDevice) and \ + not iface.parent.numa_node is None: + self.numa_node = iface.parent.numa_node + break + if self.numa_node is None or self.core is None: + self.numa_node, self.core = \ + numa_mgr.get_numa_core(numa_node = self.numa_node) + + dpdk_list = list() + + # On numa architecture socket-mem requires to set the amount of memory + # to be reserved on each numa node + socket_mem_str = 'socket-mem ' + for numa in range (0,numa_mgr.get_number_of_numa()): + if numa in socket_mem: + socket_mem_str = socket_mem_str + str(socket_mem[numa]) + else: + socket_mem_str = socket_mem_str + '0' + + if numa < numa_mgr.get_number_of_numa()-1: + socket_mem_str = socket_mem_str + ',' + + dpdk_list.append(socket_mem_str) + + for interface in self.node.interfaces: + if isinstance(interface, DpdkDevice): + dpdk_list.append('dev ' + interface.pci_address) + + # Add the core on which running vpp and the dpdk parameters + setup = TPL_VPP_DPDK_DAEMON_SCRIPT + 'cpu {' + + setup = setup + ''' \n main-core ''' + str(self.core) + + if self.enable_worker: + self.numa_node, cpu_worker =numa_mgr.get_numa_core(self.numa_node) + setup = setup + '''\n corelist-workers ''' + str(cpu_worker) + + setup = setup + '''\n}\n\n dpdk { ''' + + for dpdk_dev in dpdk_list: + setup = setup + ''' \n ''' + dpdk_dev + + setup = setup + '\n}' + + + if any([isinstance(interface,DpdkDevice) for interface in self.node.interfaces]): + self.dpdk_setup_file.content = setup + else: + self.dpdk_setup_file.content = TPL_VPP_DPDK_DAEMON_SCRIPT + + lock = self.node.node_with_kernel.vpp_host.vppstart_lock + + vpp_disable = BashTask(self.node, CMD_VPP_DISABLE, lock = lock) + vpp_stop = BashTask(self.node, CMD_VPP_STOP, lock = lock) + enable_ip_forward = BashTask(self.node, CMD_DISABLE_IP_FORWARD) + start_vpp = BashTask(self.node, CMD_VPP_START, lock = lock) + + return ((vpp_disable > vpp_stop) | enable_ip_forward) > start_vpp + + def __delete__(self): + return BashTask(self.node, CMD_VPP_STOP) + + def _add_plugins(self, plugin): + return BashTask(self.node, CMD_VPP_ENABLE_PLUGIN, {'plugin': plugin}) + + def _set_plugins(self): + cmd = None + for plugin in self.plugins: + cmd = cmd > BashTask(self.node, CMD_VPP_ENABLE_PLUGIN, + {'plugin' : plugin}) + return cmd + + def _remove_plugins(self, plugin): + raise NotImplementedError + + @inline_task + def _get_plugins(self): + return {'plugins' : []} diff --git a/vicn/resource/vpp/vpp_bridge.py b/vicn/resource/vpp/vpp_bridge.py new file mode 100644 index 00000000..c7a70c02 --- /dev/null +++ b/vicn/resource/vpp/vpp_bridge.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from netmodel.model.type import Integer +from vicn.core.attribute import Attribute, Multiplicity +from vicn.core.attribute import Reference +from vicn.core.exception import ResourceNotFound +from vicn.core.requirement import Requirement +from vicn.core.resource_mgr import wait_resource_task +from vicn.core.resource import Resource +from vicn.core.task import task, BashTask, EmptyTask +from vicn.resource.channel import Channel +from vicn.resource.linux.application import LinuxApplication +from vicn.resource.linux.sym_veth_pair import SymVethPair +from vicn.resource.linux.sym_veth_pair import SymVethPair +from vicn.resource.node import Node +from vicn.resource.vpp.dpdk_device import DpdkDevice +from vicn.resource.vpp.interface import VPPInterface +from vicn.resource.vpp.vpp import VPP + +CMD_ADD_INTERFACE_TO_BR = ('vppctl set interface l2 bridge ' + '{interface.device_name} {br_domain}') + +class VPPBridge(Channel, LinuxApplication): + """ + Resource: VPPBridge + + VPPBridge instantiate a vpp resource and set it as a vpp bridge. + + This resource requires to be run within a LxcContainer which will have VPP. + Every interface in the lxc_container (i.e., the ones contained in + self.node.interfaces) will be added to the vpp bridge. To connect other vpp + node to the bridge, the corresponding dpdkdevice must be added as an + interface to the channel. + """ + + # The vpp bridge _USES_ a VPP forwarder on the node + node = Attribute(Node, mandatory=True, + description = 'Node on which vpp is running', + requirements = [Requirement('vpp')]) + + connected_nodes = Attribute(Node, multiplicity = Multiplicity.OneToMany, + description = 'List of nodes to connect to the bridge') + + #-------------------------------------------------------------------------- + # Constructor and Accessors + #-------------------------------------------------------------------------- + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._vpp_interfaces = list() + + #-------------------------------------------------------------------------- + # Resource lifecycle + #-------------------------------------------------------------------------- + + def __subresources__ (self): + # We don't need any reference to the list of SymVethPair because each + # side of a veth will be included in the node.interfaces list + self._veths = [SymVethPair(node1 = self.node, node2 = node, + owner = self) for node in self.connected_nodes] + + return Resource.__concurrent__(*self._veths) + + @task + def __initialize__ (self): + # Add the veth side on the connected_nodes to the set of interfaces of + # the channel + self.interfaces.extend([veth.side2 for veth in self._veths]) + + @task + def __get__(self): + # Forces creation + raise ResourceNotFound + + # Nothing to do + __delete__ = None + + def __create__(self): + manager = self._state.manager + + # Create a VPPInterface for each interface in the node. These will be + # the interfaces we will connect to the vpp bridge process + vpp_interfaces = list() + for interface in self.node.interfaces: + # FIXME harcoded value + if interface.device_name == 'eth0': + continue + + vpp_interface = VPPInterface(vpp = self.node.vpp, + parent = interface, + ip_address = Reference(interface, 'ip_address'), + device_name = 'host-' + interface.device_name) + vpp_interfaces.append(vpp_interface) + manager.commit_resource(vpp_interface) + + tasks = EmptyTask() + + for vpp_interface in vpp_interfaces: + tasks = tasks > (wait_resource_task(vpp_interface) > + self._add_interface(vpp_interface,0)) + + return wait_resource_task(self.node.vpp) > tasks + + #-------------------------------------------------------------------------- + # Internal methods + #-------------------------------------------------------------------------- + + def _add_interface(self, interface, br_domain): + return BashTask(self.node, CMD_ADD_INTERFACE_TO_BR, + {'interface': interface, 'br_domain': br_domain}) + + def _del_interface(self, interface, br_domain): + raise NotImplementedError('Interface removal not supported') + diff --git a/vicn/resource/vpp/vpp_commands.py b/vicn/resource/vpp/vpp_commands.py new file mode 100644 index 00000000..8ee64bf6 --- /dev/null +++ b/vicn/resource/vpp/vpp_commands.py @@ -0,0 +1,41 @@ +##### VPP SETUP ##### + +CMD_VPP_STOP_SERVICE = 'systemctl stop vpp.service' +CMD_VPP_DISABLE = 'systemctl disable vpp.service' + +# 'sleep 1' ensures that VPP has enough time to start +CMD_VPP_START = ''' +systemctl start vpp +sleep 1 +''' +CMD_VPP_STOP = ''' +systemctl stop vpp +killall -9 vpp_main || true +''' +CMD_VPP_ENABLE_PLUGIN = 'vppctl {plugin} enable' + +##### VPP INTERFACES ##### + +CMD_VPP_CREATE_IFACE = ''' +# Create vpp interface from netmodel.network.interface.device_name} with mac {self.parent.mac_address} +vppctl create host-interface name {vpp_interface.parent.device_name} hw-addr {vpp_interface.parent.mac_address} +vppctl set interface state {vpp_interface.device_name} up +''' +CMD_VPP_SET_IP = 'vppctl set int ip address {netdevice.device_name} {netdevice.ip_address}/{netdevice.prefix_len}' +CMD_VPP_SET_UP = 'vppctl set int state {netdevice.device_name} up' + +##### VPP IP ROUTING ##### + +CMD_VPP_ADD_ROUTE = 'vppctl set ip arp static {route.interface.vppinterface.device_name} {route.ip_address} {route.mac_address}' +CMD_VPP_DEL_ROUTE = 'vppctl set ip arp del static {route.interface.vppinterface.device_name} {route.ip_address} {route.mac_address}' +CMD_VPP_ADD_ROUTE_GW = 'vppctl ip route add {route.ip_address}/32 via {route.gateway} {route.interface.vppinterface.device_name}' +CMD_VPP_DEL_ROUTE_GW = 'vppctl ip route del {route.ip_address}/32 via {route.gateway} {route.interface.vppinterface.device_name}' + +##### VPP CICN PLUGIN ##### + +CMD_VPP_CICN_GET = "timeout 1 vppctl cicn show" #We timeout if vpp is not started +CMD_VPP_ADD_ICN_ROUTE = 'vppctl cicn cfg fib add prefix {route.prefix} face {route.face.id}' +CMD_VPP_ADD_ICN_FACE = 'vppctl cicn cfg face add local {face.src_ip}:{face.src_port} remote {face.dst_ip}:{face.dst_port}' + +CMD_VPP_CICN_GET_CACHE_SIZE = 'vppctl cicn show | grep "CS entries" | grep -E "[0-9]+"' +CMD_VPP_CICN_SET_CACHE_SIZE = 'vppctl cicn control param cs size {self.cache_size}' diff --git a/vicn/resource/vpp/vpp_host.py b/vicn/resource/vpp/vpp_host.py new file mode 100644 index 00000000..134e65b0 --- /dev/null +++ b/vicn/resource/vpp/vpp_host.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Cisco and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio + +from netmodel.model.type import String +from vicn.core.attribute import Attribute, Multiplicity +from vicn.core.exception import ResourceNotFound +from vicn.core.requirement import Requirement +from vicn.core.task import BashTask, task, EmptyTask +from vicn.resource.linux.application import LinuxApplication +from vicn.resource.linux.file import TextFile +from vicn.resource.node import Node +from vicn.resource.vpp.scripts import FN_APPARMOR_DPDK_SCRIPT +from vicn.resource.vpp.scripts import TPL_APPARMOR_DPDK_SCRIPT +from vicn.resource.vpp.scripts import FN_VPP_DPDK_SCRIPT +from vicn.resource.vpp.scripts import TPL_VPP_DPDK_DAEMON_SCRIPT +from vicn.resource.vpp.vpp_commands import CMD_VPP_DISABLE +from vicn.resource.vpp.vpp_commands import CMD_VPP_STOP_SERVICE + +CMD_INSERT_MODULES = 'modprobe uio && modprobe igb_uio' +CMD_APP_ARMOR_RELOAD = ''' +# Force apparmor to reload profiles to include the new profile +/etc/init.d/apparmor reload +''' +CMD_SYSCTL_HUGEPAGES = 'sysctl -w vm.nr_hugepages={nb_hp}' +DEFAULT_NB_HUGEPAGES = 1024 +CMD_GREP_UIO_DEV = 'ls /dev | grep uio' +CMD_CREATE_UIO_DEVICES = "dpdk_nic_bind --bind=igb_uio {pci_address}" + +class VPPHost(LinuxApplication): + """ + Resource: VPPHost + + Only used for container deployment + + Packages required on the host + - vpp : sysctl configuration + - vpp-dpdk-dkms : kernel modules + + Host must be configured to let vpp to work into container: + - install new apparmor profile (to let the container to read + hugepages info in /sys/kernel/mm/hugepages) + - set hugepages into the host + """ + + node = Attribute(Node, + description = 'Node on which the application is installed', + mandatory = True, + multiplicity = Multiplicity.OneToOne, + reverse_name = 'vpp_host', + reverse_description = 'Setup for hosting vpp containers', + reverse_auto = True, + requirements = [Requirement('numa_mgr')]) + uio_devices = Attribute(String, + description = 'uio devices on the node', + multiplicity = Multiplicity.OneToMany, + ro = True) + dpdk_devices = Attribute(String, + description = 'Dpdk devices on the node', + multiplicity = Multiplicity.OneToMany) + + __package_names__ = ['dpdk', 'vpp', 'vpp-dpdk-dkms'] + + #-------------------------------------------------------------------------- + # Constructor and Accessors + #-------------------------------------------------------------------------- + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vppstart_lock = asyncio.Lock() + + #-------------------------------------------------------------------------- + # Resource lifecycle + #-------------------------------------------------------------------------- + + def __subresources__(self): + app_armor_file = TextFile(node = self.node, + filename = FN_APPARMOR_DPDK_SCRIPT, + content = TPL_APPARMOR_DPDK_SCRIPT, + overwrite = True) + startup_conf = TextFile(node = self.node, + filename = FN_VPP_DPDK_SCRIPT, + content = TPL_VPP_DPDK_DAEMON_SCRIPT, + overwrite = True) + return app_armor_file | startup_conf + + @task + def __get__(self): + """ + This method always assumes the resource does not exist, since it is not + an issue to perform the modprobe call everytime. + """ + raise ResourceNotFound + + def __create__(self): + modules = BashTask(self.node, CMD_INSERT_MODULES) + app_armor_reload = BashTask(self.node, CMD_APP_ARMOR_RELOAD) + sysctl_hugepages = BashTask(self.node, CMD_SYSCTL_HUGEPAGES, + {'nb_hp': DEFAULT_NB_HUGEPAGES}) + + # Hook + # The following is needed to create uio devices in /dev. They are + # required to let vpp to use dpdk (or other compatibles) nics. From a + # container, vpp cannot create those devices, therefore we need to + # create them in the host and then mount them on each container running + # vpp (and using a physical nic) + stop_vpp = BashTask(self.node, CMD_VPP_STOP_SERVICE) + disable_vpp = BashTask(self.node, CMD_VPP_DISABLE) + disable_vpp = stop_vpp > disable_vpp + + create_uio = EmptyTask() + for device in self.dpdk_devices: + create_uio = create_uio > BashTask(self.node, + CMD_CREATE_UIO_DEVICES, {'pci_address' : device}) + + return ((modules | app_armor_reload) | sysctl_hugepages) > \ + (disable_vpp > create_uio) + + __delete__ = None + + #-------------------------------------------------------------------------- + # Attributes + #-------------------------------------------------------------------------- + + def _get_uio_devices(self): + def parse(rv): + return rv.stdout.splitlines() + return BashTask(self.node, CMD_GREP_UIO_DEV, parse = parse) |