aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarcel Enguehard <mengueha+fdio@cisco.com>2017-05-23 10:50:17 +0200
committerMarcel Enguehard <mengueha+fdio@cisco.com>2017-05-23 08:57:12 +0000
commitbe0b435d307173598c30fcacc421b17112137099 (patch)
treeed8aae7eef796eb7381d21937b095a036733789a
parent895a6328d6e64948ed213e8fbbb3ab15aca0df43 (diff)
Introduced groups + lxd profiles + diverted control network handling to lxd + misc bug fixes
Change-Id: Iae26bc2994ac9704dde7dfa8fbe4be1b74cf9e6f Signed-off-by: Marcel Enguehard <mengueha+fdio@cisco.com>
-rwxr-xr-xconfig/logging.conf2
-rw-r--r--examples/tutorial/tutorial01.json25
-rw-r--r--netmodel/model/collection.py (renamed from vicn/resource/gui.py)15
-rw-r--r--netmodel/util/file.py74
-rw-r--r--netmodel/util/log.py12
-rwxr-xr-xvicn/bin/vicn.py40
-rw-r--r--vicn/core/api.py72
-rw-r--r--vicn/core/attribute.py19
-rw-r--r--vicn/core/collection.py23
-rw-r--r--vicn/core/exception.py7
-rw-r--r--vicn/core/resource.py265
-rw-r--r--vicn/core/resource_mgr.py58
-rw-r--r--vicn/core/sa_collections.py25
-rw-r--r--vicn/core/state.py9
-rw-r--r--vicn/core/task.py8
-rw-r--r--vicn/resource/central.py280
-rw-r--r--vicn/resource/channel.py25
-rw-r--r--vicn/resource/group.py38
-rw-r--r--vicn/resource/icn/ndnpingserver.py2
-rw-r--r--vicn/resource/icn/webserver.py4
-rw-r--r--vicn/resource/ip_assignment.py37
-rw-r--r--vicn/resource/linux/application.py1
-rw-r--r--vicn/resource/linux/bridge.py2
-rw-r--r--vicn/resource/linux/dnsmasq.py10
-rw-r--r--vicn/resource/linux/net_device.py16
-rw-r--r--vicn/resource/linux/package_manager.py55
-rw-r--r--vicn/resource/linux/repository.py7
-rw-r--r--vicn/resource/lxd/lxc_container.py89
-rw-r--r--vicn/resource/lxd/lxd_hypervisor.py45
-rw-r--r--vicn/resource/lxd/lxd_profile.py56
-rw-r--r--vicn/resource/node.py26
31 files changed, 809 insertions, 538 deletions
diff --git a/config/logging.conf b/config/logging.conf
index b1ca30fd..b21a4c4d 100755
--- a/config/logging.conf
+++ b/config/logging.conf
@@ -15,7 +15,7 @@ handlers=file_handler
class=FileHandler
level=DEBUG
formatter=formatter
-args=("/tmp/vicn.log", "w")
+args=("~/.vicn/vicn.log", "w")
[formatter_formatter]
format=%(asctime)s %(levelname)8s %(name)25s.%(funcName)25s %(message)20s
diff --git a/examples/tutorial/tutorial01.json b/examples/tutorial/tutorial01.json
index edc0e6da..dbef980f 100644
--- a/examples/tutorial/tutorial01.json
+++ b/examples/tutorial/tutorial01.json
@@ -1,6 +1,10 @@
{
"resources": [
{
+ "type": "Group",
+ "name": "topology"
+ },
+ {
"type": "Physical",
"name": "server",
"hostname": "localhost"
@@ -14,41 +18,48 @@
{
"type": "LxcImage",
"name": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"node": "server"
},
{
"type": "LxcContainer",
"image": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"name": "prod1",
"node": "server"
},
{
"type": "LxcContainer",
"image": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"name": "prod2",
"node": "server"
},
{
"type": "LxcContainer",
"image": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"name": "core2",
"node": "server"
},
{
"type": "LxcContainer",
"image": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"name": "core1",
"node": "server"
},
{
"type": "LxcContainer",
"image": "ubuntu1604-cicnsuite-rc2",
+ "groups": ["topology"],
"name": "cons1",
"node": "server"
},
{
"type": "LxcContainer",
"name": "cons2",
+ "groups": ["topology"],
"node": "server",
"image": "ubuntu1604-cicnsuite-rc2"
},
@@ -60,7 +71,7 @@
"type": "WebServer",
"node": "prod1",
"prefixes": [
- "/webserver"
+ "/webserver1"
]
},
{
@@ -71,7 +82,7 @@
"type": "WebServer",
"node": "prod2",
"prefixes": [
- "/webserver"
+ "/webserver2"
]
},
{
@@ -92,26 +103,31 @@
},
{
"type": "Link",
+ "groups": ["topology"],
"src_node": "cons1",
"dst_node": "core1"
},
{
"type": "Link",
+ "groups": ["topology"],
"src_node": "cons2",
"dst_node": "core1"
},
{
"type": "Link",
+ "groups": ["topology"],
"src_node": "core1",
"dst_node": "core2"
},
{
"type": "Link",
+ "groups": ["topology"],
"src_node": "core2",
"dst_node": "prod1"
},
{
"type": "Link",
+ "groups": ["topology"],
"src_node": "core2",
"dst_node": "prod2"
},
@@ -120,11 +136,12 @@
"ip_routing_strategy": "spt",
"ip6_data_prefix": "2001::/50",
"ip4_data_prefix": "192.168.128.0/24",
- "ip4_control_prefix": "192.168.140.0/24"
+ "groups": ["topology"]
},
{
"type": "CentralICN",
- "face_protocol": "udp4"
+ "face_protocol": "udp4",
+ "groups": ["topology"]
}
]
}
diff --git a/vicn/resource/gui.py b/netmodel/model/collection.py
index 3ded7a5a..21be84d8 100644
--- a/vicn/resource/gui.py
+++ b/netmodel/model/collection.py
@@ -16,14 +16,13 @@
# limitations under the License.
#
-from vicn.helpers.resource_definition import *
+from netmodel.model.filter import Filter
-class GUI(Resource):
+class Collection(list):
"""
- Resource: GUI
-
- This resource is empty on purpose. It is a temporary resource used as a
- placeholder for controlling the GUI and should be deprecated in future
- releases.
+ A collection corresponds to a list of objects, and includes processing functionalities to
+ manipulate them.
"""
- pass
+
+ def filter(self, filter):
+ return filter.filter(self)
diff --git a/netmodel/util/file.py b/netmodel/util/file.py
new file mode 100644
index 00000000..4204d533
--- /dev/null
+++ b/netmodel/util/file.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import errno
+import logging
+import os
+import tempfile
+
+log = logging.getLogger(__name__)
+
+def mkdir(directory):
+ """
+ Create a directory (mkdir -p).
+ Args:
+ directory: A String containing an absolute path.
+ Raises:
+ OSError: If the directory cannot be created.
+ """
+ try:
+ if not os.path.exists(directory):
+ log.info("Creating '%s' directory" % directory)
+ os.makedirs(directory)
+ except OSError as e:
+ if e.errno == errno.EEXIST and os.path.isdir(directory):
+ pass
+ else:
+ raise OSError("Cannot mkdir %s: %s" % (directory, e))
+
+def check_writable_directory(directory):
+ """
+ Tests whether a directory is writable.
+ Args:
+ directory: A String containing an absolute path.
+ Raises:
+ RuntimeError: If the directory does not exists or isn't writable.
+ """
+ if not os.path.exists(directory):
+ raise RuntimeError("Directory '%s' does not exists" % directory)
+ if not os.access(directory, os.W_OK | os.X_OK):
+ raise RuntimeError("Directory '%s' is not writable" % directory)
+ try:
+ with tempfile.TemporaryFile(dir = directory):
+ pass
+ except Exception as e:
+ raise RuntimeError("Cannot write into directory '%s': %s" % (directory, e))
+
+def ensure_writable_directory(directory):
+ """
+ Tests whether a directory exists and is writable. If not,
+ try to create such a directory.
+ Args:
+ directory: A String containing an absolute path.
+ Raises:
+ RuntimeError: If the directory does not exists and cannot be created.
+ """
+ try:
+ check_writable_directory(directory)
+ except RuntimeError as e:
+ mkdir(directory)
diff --git a/netmodel/util/log.py b/netmodel/util/log.py
index 68eb9a7f..f9fa1e03 100644
--- a/netmodel/util/log.py
+++ b/netmodel/util/log.py
@@ -21,6 +21,17 @@ import logging.config
import os
import sys
+from netmodel.util.file import ensure_writable_directory
+
+# Monkey-patch logging.FileHandler to support expanduser()
+oldFileHandler = logging.FileHandler
+class vICNFileHandler(oldFileHandler):
+ def __init__(self, filename, mode='a', encoding=None, delay=False):
+ filename = os.path.expanduser(filename)
+ ensure_writable_directory(os.path.dirname(filename))
+ super().__init__(filename, mode, encoding, delay)
+logging.FileHandler = vICNFileHandler
+
colors = {
'white': "\033[1;37m",
'yellow': "\033[1;33m",
@@ -107,6 +118,7 @@ def initialize_logging():
if os.path.exists(config_path):
logging.config.fileConfig(config_path, disable_existing_loggers=False)
+
root = logging.getLogger()
root.setLevel(logging.DEBUG)
diff --git a/vicn/bin/vicn.py b/vicn/bin/vicn.py
index 9a43cf6d..7ece629b 100755
--- a/vicn/bin/vicn.py
+++ b/vicn/bin/vicn.py
@@ -46,36 +46,34 @@ class VICNDaemon(Daemon):
n_times = 1
background = False
setup = False
- scenario = None
- node_list, net, ndn, mob, cluster = None, None, None, None, None
parser = ArgumentParser(description=textcolor('green', "Batch usage of VICN."))
- parser.add_argument('-s', metavar='configuration_file_path',
- help="JSON file containing the topology")
- parser.add_argument('-n', metavar='n_times', type=int, help='Execute the test multiple times')
- parser.add_argument('-x', action='store_false', help='No automatic execution')
+ parser.add_argument('-s', '--scenario', metavar='configuration_file_path',
+ action='append',
+ help="JSON file containing the topology")
+ parser.add_argument('-z', '--identifier', metavar='identifier', type=str, help='Experiment identifier')
+ parser.add_argument('-x', '--no-execute', action='store_false', help='Configure only, no automatic execution')
+ parser.add_argument('-c', '--clean', action='store_true', help='Clean deployment before setup')
+ parser.add_argument('-C', '--clean-only', action='store_true', help='Clean only')
arguments = parser.parse_args()
- args = vars(arguments)
+ scenario = arguments.scenario
+ if not scenario:
+ log.error('No scenario specified')
+ sys.exit(-1)
- for option in args.keys():
- if args[option] is not None:
- if option == "s":
- print(" * Loading the configuration file at {0}".format(args[option]))
- scenario = args[option]
- elif option == "t" and args[option] is True:
- background = True
- elif option == "x" and args[option] is True:
- setup = True
- elif option == "n":
- n_times = args[option]
+ identifier = arguments.identifier or "default"
+ clean = arguments.clean or arguments.clean_only
+ execute = not arguments.clean_only or arguments.no_execute
self._api = API()
- self._api.configure(scenario, setup)
+ self._api.configure(scenario)
- if node_list is not None:
- ResourceManager().set(node_list)
+ if clean:
+ self._api.teardown()
+ if execute:
+ self._api.setup(commit = True)
def main(self):
"""
diff --git a/vicn/core/api.py b/vicn/core/api.py
index 09167aa0..708e2581 100644
--- a/vicn/core/api.py
+++ b/vicn/core/api.py
@@ -33,6 +33,7 @@ from vicn.resource.node import Node
DEFAULT_SETTINGS = {
'network': '192.168.0.0/16',
+ 'bridge_name': 'br0',
'mac_address_base': '0x00163e000000',
'websocket_port': 9999
}
@@ -48,48 +49,25 @@ class Event_ts(asyncio.Event):
class API(metaclass = Singleton):
def terminate(self):
+ # XXX not valid if nothing has been initialized
ResourceManager().terminate()
- def parse_topology_file(self, topology_fn):
- log.debug("Parsing topology file %(topology_fn)s" % locals())
+ def parse_topology_file(self, topology_fn, resources, settings):
+ log.info("Parsing topology file %(topology_fn)s" % locals())
try:
topology_fd = open(topology_fn, 'r')
except IOError:
- self.error("Topology file '%(topology_fn)s not found" % locals())
- return None
+ log.error("Topology file '%(topology_fn)s not found" % locals())
+ sys.exit(1)
try:
topology = json.loads(topology_fd.read())
# SETTING
- settings = DEFAULT_SETTINGS
settings.update(topology.get('settings', dict()))
- # VICN process-related initializations
- nofile = settings.get('ulimit-n', None)
- if nofile is not None and nofile > 0:
- if nofile < 1024:
- log.error('Too few allowed open files for the process')
- import os; os._exit(1)
-
- log.info('Setting open file descriptor limit to {}'.format(
- nofile))
- ulimit.setrlimit(
- ulimit.RLIMIT_NOFILE,
- (nofile, nofile))
-
- ResourceManager(base=topology_fn, settings=settings)
-
# NODES
- resources = topology.get('resources', list())
- for resource in resources:
- try:
- ResourceManager().create_from_dict(**resource)
- except Exception as e:
- log.warning("Could not create resource '%r': %r" % \
- (resource, e,))
- import traceback; traceback.print_exc()
- continue
+ resources.extend(topology.get('resources', list()))
except SyntaxError:
log.error("Error reading topology file '%s'" % (topology_fn,))
@@ -97,16 +75,42 @@ class API(metaclass = Singleton):
log.debug("Done parsing topology file %(topology_fn)s" % locals())
- def configure(self, name, setup=False):
+ def configure(self, scenario_list):
log.info("Parsing configuration file", extra={'category': 'blue'})
- self.parse_topology_file(name)
+ resources = list()
+ settings = DEFAULT_SETTINGS
+ for scenario in scenario_list:
+ self.parse_topology_file(scenario, resources, settings)
+
+ # VICN process-related initializations
+ nofile = settings.get('ulimit-n', None)
+ if nofile is not None and nofile > 0:
+ if nofile < 1024:
+ log.error('Too few allowed open files for the process')
+ import os; os._exit(1)
+
+ log.info('Setting open file descriptor limit to {}'.format(
+ nofile))
+ ulimit.setrlimit(
+ ulimit.RLIMIT_NOFILE,
+ (nofile, nofile))
+
+ ResourceManager(base=scenario[-1], settings=settings)
+
+ for resource in resources:
+ try:
+ ResourceManager().create_from_dict(**resource)
+ except Exception as e:
+ log.error("Could not create resource '%r': %r" % \
+ (resource, e,))
+ import os; os._exit(1)
+
self._configured = True
- ResourceManager().setup(commit=setup)
- def setup(self):
+ def setup(self, commit = False):
if not self._configured:
raise NotConfigured
- ResourceManager().setup()
+ ResourceManager().setup(commit)
def teardown(self):
ResourceManager().teardown()
diff --git a/vicn/core/attribute.py b/vicn/core/attribute.py
index f6ec7c70..22f44487 100644
--- a/vicn/core/attribute.py
+++ b/vicn/core/attribute.py
@@ -42,7 +42,6 @@ class Multiplicity:
OneToMany = '1_N'
ManyToOne = 'N_1'
ManyToMany = 'N_N'
-
@staticmethod
def reverse(value):
@@ -108,7 +107,7 @@ class Attribute(abc.ABC, ObjectSpecification):
self.is_aggregate = False
self._reverse_attributes = list()
-
+
#--------------------------------------------------------------------------
# Display
#--------------------------------------------------------------------------
@@ -157,7 +156,7 @@ class Attribute(abc.ABC, ObjectSpecification):
value = value.get_uuid()
return value
else:
- try:
+ try:
cur_value = vars(instance)[self.name]
if self.is_collection:
# copy the list
@@ -167,11 +166,11 @@ class Attribute(abc.ABC, ObjectSpecification):
if self.is_collection:
cur_value = list()
- instance._state.dirty[self.name].trigger(Operations.LIST_ADD,
+ instance._state.dirty[self.name].trigger(Operations.LIST_ADD,
value, cur_value)
# prevent instrumented list to perform operation
- raise VICNListException
+ raise VICNListException
def do_list_remove(self, instance, value):
if instance.is_local_attribute(self.name):
@@ -184,11 +183,11 @@ class Attribute(abc.ABC, ObjectSpecification):
if self.is_collection:
# copy the list
cur_value = list(cur_value)
- instance._state.dirty[self.name].trigger(Operations.LIST_REMOVE,
+ instance._state.dirty[self.name].trigger(Operations.LIST_REMOVE,
value, cur_value)
# prevent instrumented list to perform operation
- raise VICNListException
+ raise VICNListException
def do_list_clear(self, instance):
if instance.is_local_attribute(self.name):
@@ -198,11 +197,11 @@ class Attribute(abc.ABC, ObjectSpecification):
if self.is_collection:
# copy the list
cur_value = list(cur_value)
- instance._state.dirty[self.name].trigger(Operations.LIST_CLEAR,
+ instance._state.dirty[self.name].trigger(Operations.LIST_CLEAR,
value, cur_value)
# prevent instrumented list to perform operation
- raise VICNListException
+ raise VICNListException
def handle_getitem(self, instance, item):
if isinstance(item, UUID):
@@ -227,7 +226,7 @@ class Attribute(abc.ABC, ObjectSpecification):
@property
def is_collection(self):
- return self.multiplicity in (Multiplicity.OneToMany,
+ return self.multiplicity in (Multiplicity.OneToMany,
Multiplicity.ManyToMany)
def is_set(self, instance):
diff --git a/vicn/core/collection.py b/vicn/core/collection.py
new file mode 100644
index 00000000..fb222891
--- /dev/null
+++ b/vicn/core/collection.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from vicn.core.sa_collections import InstrumentedList
+from netmodel.model.collection import Collection
+
+class Collection(InstrumentedList, Collection):
+ pass
diff --git a/vicn/core/exception.py b/vicn/core/exception.py
index d7422723..977fc8ad 100644
--- a/vicn/core/exception.py
+++ b/vicn/core/exception.py
@@ -37,3 +37,10 @@ class SetupException(VICNException): pass
class VICNListException(VICNException): pass
class ResourceNotFound(VICNException): pass
+
+class VICNWouldBlock(VICNException):
+ """
+ Exception called when a request would block and the user explicitely
+ required non-blocking behaviour
+ """
+ pass
diff --git a/vicn/core/resource.py b/vicn/core/resource.py
index 9355cd07..ab96daa5 100644
--- a/vicn/core/resource.py
+++ b/vicn/core/resource.py
@@ -27,6 +27,10 @@ import traceback
import types
from abc import ABC, ABCMeta
+from threading import Event as ThreadEvent
+
+# LXD workaround
+from pylxd.exceptions import NotFound as LXDAPIException
from netmodel.model.mapper import ObjectSpecification
from netmodel.model.type import String, Bool, Integer, Dict
@@ -35,12 +39,13 @@ from netmodel.util.deprecated import deprecated
from netmodel.util.singleton import Singleton
from vicn.core.attribute import Attribute, Multiplicity, Reference
from vicn.core.attribute import NEVER_SET
+from vicn.core.collection import Collection
from vicn.core.commands import ReturnValue
from vicn.core.event import Event, AttributeChangedEvent
from vicn.core.exception import VICNException, ResourceNotFound
+from vicn.core.exception import VICNWouldBlock
from vicn.core.resource_factory import ResourceFactory
from vicn.core.requirement import Requirement, Property
-from vicn.core.sa_collections import InstrumentedList, _list_decorators
from vicn.core.scheduling_algebra import SchedulingAlgebra
from vicn.core.state import ResourceState, UUID
from vicn.core.state import Operations, InstanceState
@@ -95,12 +100,24 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
The base Resource class implements all the logic related to resource
instances.
-
- See also :
+
+ See also :
* ResourceManager : logic related to class instanciation
* Resource metaclass : logic related to class construction
* ResourceFactory : logic related to available classes and mapping from
name to type
+
+ Internal attributes:
+
+ - _reverse_attributes: a dict mapping attribute objects with the class
+ that declared the reverse attribute.
+
+ For instance, a Group declares a collection of Resource objects through
+ its resources attributes. It also mentions a reverse attribute named
+ 'groups'. This means every Resource class will be equipped with a
+ groups attribute, being a collection of Group objects.
+
+ Resource._reverse_attributes = { <Attribute: groups> : Resource }
"""
__type__ = TopLevelResource
@@ -139,7 +156,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
# Cache dependencies
self._deps = None
-
+
# Internal data tag for resources
self._internal_data = dict()
@@ -168,7 +185,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
else:
resource = x
if not resource:
- raise VICNException(E_UNK_RES_NAME.format(key,
+ raise VICNException(E_UNK_RES_NAME.format(key,
self.name, self.__class__.__name__, x))
element = resource if isinstance(resource, Reference) \
else resource._state.uuid
@@ -176,13 +193,13 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
value = new_value
else:
if isinstance(value, str):
- resource = ResourceManager().by_name(value)
+ resource = ResourceManager().by_name(value)
elif isinstance(value, UUID):
- resource = ResourceManager().by_uuid(value)
+ resource = ResourceManager().by_uuid(value)
else:
resource = value
if not resource:
- raise VICNException(E_UNK_RES_NAME.format(key,
+ raise VICNException(E_UNK_RES_NAME.format(key,
self.name, self.__class__.__name__, value))
value = value if isinstance(resource, Reference) \
else resource._state.uuid
@@ -202,7 +219,6 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
default = self.get_default_collection(attr) if attr.is_collection else \
self.get_default(attr)
if vars(attr)['default'] != NEVER_SET:
- #import pdb; pdb.set_trace()
self.set(attr.name, default, blocking=False)
if issubclass(attr.type, Resource) and attr.requirements:
for req in attr.requirements:
@@ -218,7 +234,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
def __after_init__(self):
return tuple()
-
+
def __subresources__(self):
return None
@@ -248,8 +264,8 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
attribute = self.get_attribute(attribute_name)
- # Handling Lambda attributes
if hasattr(attribute, 'func') and attribute.func:
+ # Handling Lambda attributes
value = attribute.func(self)
else:
if self.is_local_attribute(attribute.name):
@@ -266,25 +282,44 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
if value is NEVER_SET:
if not allow_never_set:
- log.error(E_GET_NON_LOCAL.format(attribute_name,
+ log.error(E_GET_NON_LOCAL.format(attribute_name,
self._state.uuid))
raise NotImplementedError
- if attribute.is_collection:
- value = self.get_default_collection(attribute)
- else:
- if attribute.auto:
- # Automatic instanciation
- if attribute.requirements:
- log.warning('Ignored requirements {}'.format(
- attribute.requirements))
- value = self.auto_instanciate(attribute)
-
- if value is NEVER_SET:
- value = self.get_default(attribute)
-
- if self.is_local_attribute(attribute.name):
- self.set(attribute.name, value)
+ # node.routing_table is local and auto, so this needs to be tested first...
+ if attribute.auto:
+ # Automatic instanciation
+ #
+ # Used for instance in route.node.routing_table.routes
+ if attribute.requirements:
+ log.warning('Ignored requirements {}'.format(
+ attribute.requirements))
+ value = self.auto_instanciate(attribute)
+
+ if value is NEVER_SET:
+ if self.is_local_attribute(attribute.name):
+ if attribute.is_collection:
+ value = self.get_default_collection(attribute)
+ else:
+ value = self.get_default(attribute)
+ self.set(attribute.name, value)
+ else:
+ log.info("Fetching remote value for {}.{}".format(self,attribute.name))
+ task = getattr(self, "_get_{}".format(attribute.name))()
+ #XXX This is ugly but it prevents the LxdNotFound exception
+ while True:
+ try:
+ rv = task.execute_blocking()
+ break
+ except LxdAPIException:
+ log.warning("LxdAPIException, retrying to fetch value")
+ continue
+ except Exception as e:
+ import traceback; traceback.print_tb(e.__traceback__)
+ log.error("Failed to retrieve remote value for {} on {}".format(attribute.name, self))
+ import os; os._exit(1)
+ value = rv[attribute.name]
+ vars(self)[attribute.name] = value
if unref and isinstance(value, UUID):
value = self.from_uuid(value)
@@ -297,6 +332,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
return value
+ # XXX async_get should not be blocking
async def async_get(self, attribute_name, default=NEVER_SET, unref=True,
resolve=True, allow_never_set=False, blocking=True):
attribute = self.get_attribute(attribute_name)
@@ -318,14 +354,14 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
# exists
value = vars(self).get(attribute.name, NEVER_SET)
if value is NEVER_SET:
- await self._state.manager.attribute_get(self,
+ await self._state.manager.attribute_get(self,
attribute_name, value)
value = vars(self).get(attribute.name, NEVER_SET)
# Handling NEVER_SET
if value is NEVER_SET:
if not allow_never_set:
- log.error(E_GET_NON_LOCAL.format(attribute_name,
+ log.error(E_GET_NON_LOCAL.format(attribute_name,
self._state.uuid))
raise NotImplementedError
@@ -366,27 +402,19 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
"""
attribute = self.get_attribute(attribute_name)
- if set_reverse and attribute.reverse_name:
+ # Let's transform value if not in the proper format
+ if attribute.is_collection and not isinstance(value, Collection):
+ value = Collection.from_list(value, self, attribute)
+ else:
+ if isinstance(value, UUID):
+ value = self.from_uuid(value)
+
+ if set_reverse and attribute.reverse_name:
for base in self.__class__.mro():
if not hasattr(base, '_reverse_attributes'):
continue
for ra in base._reverse_attributes.get(attribute, list()):
- # Value information : we need resources, not uuids
- if attribute.is_collection:
- lst = list()
- if value:
- for x in value:
- if isinstance(x, UUID):
- x = self.from_uuid(x)
- lst.append(x)
- value = InstrumentedList(lst)
- value._attribute = attribute
- value._instance = self
- else:
- if isinstance(value, UUID):
- value = self.from_uuid(value)
-
if ra.multiplicity == Multiplicity.OneToOne:
if value is not None:
value.set(ra.name, self, set_reverse = False)
@@ -400,30 +428,23 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
else:
value is None
elif ra.multiplicity == Multiplicity.ManyToMany:
- collection = value.get(ra.name)
- value.extend(self)
-
- # Handling value : we need uuids, not resources
- if attribute.is_collection:
- if not isinstance(value, InstrumentedList):
- lst = list()
- if value:
- for x in value:
- if isinstance(x, Resource):
- x = x.get_uuid()
- lst.append(x)
-
- value = InstrumentedList(lst)
- else:
- value = InstrumentedList([])
- value._attribute = attribute
- value._instance = self
- else:
- if isinstance(value, Resource):
- value = value.get_uuid()
+ # Example:
+ # _set(self, attribute_name)
+ # self = Resource()
+ # attribute_name = <Attribute groups>
+ # value = <Collection 140052309461896 [<Group: topology resources=[], name=topology, owner=None, managed=True>]>
+ # element = <Group: ...>
+
+ # We add each element of the collection to the remote
+ # attribute which is also a collection
+ for element in value:
+ collection = element.get(ra.name)
+ # XXX duplicates ?
+ collection.append(self)
+
return value
- def set(self, attribute_name, value, current=False, set_reverse=True,
+ def set(self, attribute_name, value, current=False, set_reverse=True,
blocking = True):
value = self._set(attribute_name, value, current=current,
set_reverse=set_reverse)
@@ -479,14 +500,11 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
if attribute.default._resource is Self:
default = getattr(self, attribute.default._attribute)
else:
- default = getattr(attribute.default._resource,
+ default = getattr(attribute.default._resource,
attribute.default._attribute)
else:
default = attribute.default
- value = InstrumentedList(default)
- value._attribute = attribute
- value._instance = self
- return value
+ return Collection.from_list(default, self, attribute)
def get_default(self, attribute):
if isinstance(attribute.default, types.FunctionType):
@@ -495,7 +513,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
if attribute.default._resource is Self:
value = getattr(self, attribute.default._attribute)
else:
- value = getattr(attribute.default._resource,
+ value = getattr(attribute.default._resource,
attribute.default._attribute)
else:
value = copy.deepcopy(attribute.default)
@@ -525,17 +543,22 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
@classmethod
def _sanitize(cls):
- """Sanitize the object model to accomodate for multiple declaration
- styles
+ """
+ This methods performs sanitization of the object declaration.
+
+ More specifically:
+ - it goes over all attributes and sets their name based on the python
+ object attribute name.
+ - it establishes mutual object relationships through reverse attributes.
- In particular, this method:
- - set names to all attributes
"""
cls._reverse_attributes = dict()
cur_reverse_attributes = dict()
for name, obj in vars(cls).items():
if not isinstance(obj, ObjectSpecification):
continue
+
+ # XXX it seems obj should always be an attribute, confirm !
if isinstance(obj, Attribute):
obj.name = name
@@ -555,23 +578,62 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
#
# NOTE: we need to do this after merging to be sure we get all
# properties inherited from parent (eg. multiplicity)
+ #
+ # See "Reverse attributes" section in BaseResource docstring.
+ #
+ # Continueing with the same example, let's detail how it is handled:
+ #
+ # Original declaration:
+ # >>>
+ # class Group(Resource):
+ # resources = Attribute(Resource, description = 'Resources belonging to the group',
+ # multiplicity = Multiplicity.ManyToMany,
+ # default = [],
+ # reverse_name = 'groups',
+ # reverse_description = 'Groups to which the resource belongs')
+ # <<<
+ #
+ # Local variables:
+ # cls = <class 'vicn.resource.group.Group'>
+ # obj = <Attribute resources>
+ # obj.type = <class 'vicn.core.Resource'>
+ # reverse_attribute = <Attribute groups>
+ #
+ # Result:
+ # 1) Group._reverse_attributes =
+ # { <Attribute resources> : [<Attribute groups>, ...], ...}
+ # 2) Add attribute <Attribute groups> to class Resource
+ # 3) Resource._reverse_attributes =
+ # { <Attribute groups> : [<Attribute resources], ...], ...}
+ #
if has_reverse:
a = {
- 'name' : obj.reverse_name,
- 'description' : obj.reverse_description,
- 'multiplicity' : Multiplicity.reverse(obj.multiplicity),
- 'auto' : obj.reverse_auto,
+ 'name' : obj.reverse_name,
+ 'description' : obj.reverse_description,
+ 'multiplicity' : Multiplicity.reverse(obj.multiplicity),
+ 'reverse_name' : obj.name,
+ 'reverse_description' : obj.description,
+ 'auto' : obj.reverse_auto,
}
reverse_attribute = Attribute(cls, **a)
reverse_attribute.is_aggregate = True
+ # 1) Store the reverse attributes to be later inserted in the
+ # remote class, at the end of the function
+ # TODO : clarify the reasons to perform this in two steps
cur_reverse_attributes[obj.type] = reverse_attribute
- #print('*** class backref ***', cls, obj, reverse_attribute)
+ # 2)
if not obj in cls._reverse_attributes:
cls._reverse_attributes[obj] = list()
cls._reverse_attributes[obj].append(reverse_attribute)
+ # 3)
+ if not reverse_attribute in obj.type._reverse_attributes:
+ obj.type._reverse_attributes[reverse_attribute] = list()
+ obj.type._reverse_attributes[reverse_attribute].append(obj)
+
+ # Insert newly created reverse attributes in the remote class
for kls, a in cur_reverse_attributes.items():
setattr(kls, a.name, a)
@@ -583,7 +645,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
continue
if attribute.is_aggregate and not aggregates:
continue
-
+
yield attribute
def iter_keys(self):
@@ -617,7 +679,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
for req in reqs:
if req._type != attribute.name:
continue
-
+
for attr_name, prop in req.properties.items():
value = next(iter(prop.value))
capabilities |= req._capabilities
@@ -626,7 +688,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
cls = self._state.manager.get_resource_with_capabilities(
attribute.type, capabilities)
- # Before creating a new instance of a class, let's check
+ # Before creating a new instance of a class, let's check
resource = cls(**cstr_attributes)
self._state.manager.commit_resource(resource)
@@ -636,10 +698,10 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
return list(self.iter_attributes(aggregates = aggregates))
def get_attribute_names(self, aggregates = False):
- return set(a.name
+ return set(a.name
for a in self.iter_attributes(aggregates = aggregates))
- def get_attribute_dict(self, field_names = None, aggregates = False,
+ def get_attribute_dict(self, field_names = None, aggregates = False,
uuid = True):
assert not field_names or field_names.is_star()
attributes = self.get_attributes(aggregates = aggregates)
@@ -653,11 +715,11 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
ret[a.name] = list()
for x in value:
if uuid and isinstance(x, Resource):
- x = x._state.uuid._uuid
+ x = x._state.uuid._uuid
ret[a.name].append(x)
else:
if uuid and isinstance(value, Resource):
- value = value._state.uuid._uuid
+ value = value._state.uuid._uuid
ret[a.name] = value
return ret
@@ -673,7 +735,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
self._state.state = state
def get_types(self):
- return [cls.__name__.lower() for cls in self.__class__.mro()
+ return [cls.__name__.lower() for cls in self.__class__.mro()
if cls.__name__ not in ('ABC', 'BaseType', 'object')]
def get_type(self):
@@ -686,12 +748,20 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
# Showing aggregate attributes can cause infinite loops
name = self._state.uuid if self.name in (None, NEVER_SET) else self.name
return '<{}: {} {}>'.format(self.__class__.__name__, name,
- ', '.join('{}={}'.format(k,v)
- for k, v in self.get_attribute_dict().items()))
+ ', '.join('{}={}'.format(k,v)
+ for k, v in self.get_attribute_dict().items()))
def __str__(self):
return self.__repr__()
+ def to_dict(self):
+ dic = self.get_attribute_dict(aggregates = True)
+ dic['id'] = self._state.uuid._uuid
+ dic['type'] = self.get_types()
+ dic['state'] = self._state.state
+ dic['log'] = self._state.log
+ return dic
+
#---------------------------------------------------------------------------
# Resource helpers
#---------------------------------------------------------------------------
@@ -709,7 +779,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
if not value:
continue
- if a.multiplicity in (Multiplicity.OneToOne,
+ if a.multiplicity in (Multiplicity.OneToOne,
Multiplicity.ManyToOne):
resource = value
if not resource:
@@ -742,13 +812,12 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
l.extend(list(args))
if id:
N = 3
- uuid = ''.join(random.choice(string.ascii_uppercase +
+ uuid = ''.join(random.choice(string.ascii_uppercase +
string.digits) for _ in range(N))
l.append(uuid)
name = NAME_SEP.join(str(x) for x in l)
return name
-
def check_requirements(self):
for attr in self.iter_attributes():
if issubclass(attr.type, Resource) and attr.requirements:
@@ -762,7 +831,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
@deprecated
def trigger(self, action, attribute_name, *args, **kwargs):
- self._state.manager.trigger(self, action, attribute_name,
+ self._state.manager.trigger(self, action, attribute_name,
*args, **kwargs)
#--------------------------------------------------------------------------
@@ -816,7 +885,7 @@ class BaseResource(BaseType, ABC, metaclass=ResourceMetaclass):
return hasattr(self, '_{}_{}'.format(action, attribute.name))
def is_setup(self):
- return self.state in (ResourceState.SETUP_PENDING,
+ return self.state in (ResourceState.SETUP_PENDING,
ResourceState.SETUP, ResourceState.DIRTY)
__get__ = None
@@ -851,7 +920,7 @@ class CompositionMixin:
await element._state.clean.wait()
self._state.clean.set()
-_Resource, EmptyResource = SchedulingAlgebra(BaseResource, ConcurrentMixin,
+_Resource, EmptyResource = SchedulingAlgebra(BaseResource, ConcurrentMixin,
CompositionMixin, SequentialMixin)
class ManagedResource(_Resource):
diff --git a/vicn/core/resource_mgr.py b/vicn/core/resource_mgr.py
index c6ce77ab..4ca8060c 100644
--- a/vicn/core/resource_mgr.py
+++ b/vicn/core/resource_mgr.py
@@ -288,6 +288,7 @@ class ResourceManager(metaclass=Singleton):
def create_from_dict(self, **resource):
resource_type = resource.pop('type', None)
+
assert resource_type
return self.create(resource_type.lower(), **resource)
@@ -354,6 +355,21 @@ class ResourceManager(metaclass=Singleton):
if commit:
self.commit()
+ def teardown(self):
+ asyncio.ensure_future(self._teardown())
+
+ async def _teardown(self):
+ task = EmptyTask()
+ # XXX we should never have to autoinstanciate
+ # XXX why keeping this code
+ for resource in self.get_resources():
+ if resource.get_type() == 'lxccontainer':
+ task = task | resource.__delete__()
+ print("RESOURCE", resource.name)
+ self.schedule(task)
+ ret = await wait_task(task)
+ return ret
+
def get_resource_with_capabilities(self, cls, capabilities):
if '__type__' in cls.__dict__ and cls.__type__ == FactoryResource:
candidates = inheritors(cls)
@@ -806,7 +822,10 @@ class ResourceManager(metaclass=Singleton):
resource._state.attr_change_success[attribute.name] = True
else:
log.error('Attribute error {} for resource {}'.format(
- resource.get_uuid(), attribute.name))
+ attribute.name, resource.get_uuid()))
+ print("task1=", task)
+ sys.stdout.flush()
+
import traceback; traceback.print_tb(e.__traceback__)
log.error('Failed with exception: {}'.format(e))
import os; os._exit(1)
@@ -931,10 +950,13 @@ class ResourceManager(metaclass=Singleton):
resource._state.attr_change_success[attribute.name] = True
else:
log.error('Attribute error {} for resource {}'.format(
- resource.get_uuid(), attribute.name))
+ attribute.name, resource.get_uuid()))
+ # XXX need better logging
+ print("task2=", task._node.name, task.get_full_cmd())
+ sys.stdout.flush()
e = resource._state.attr_change_value[attribute.name]
- new_state = AttributeState.ERROR
import traceback; traceback.print_tb(e.__traceback__)
+ new_state = AttributeState.ERROR
import os; os._exit(1)
else:
@@ -956,9 +978,10 @@ class ResourceManager(metaclass=Singleton):
return Query.from_dict(dic)
def _monitor_netmon(self, resource):
- ip = resource.node.host_interface.ip4_address
+ ip = resource.node.management_interface.ip4_address
if not ip:
- log.error('IP of monitored Node is None')
+ log.error('IP of monitored Node {} is None'.format(resource.node))
+ #return # XXX
import os; os._exit(1)
ws = self._router.add_interface('websocketclient', address=ip,
@@ -1009,7 +1032,7 @@ class ResourceManager(metaclass=Singleton):
def _monitor_emulator(self, resource):
ns = resource
- ip = ns.node.bridge.ip4_address # host_interface.ip_address
+ ip = ns.node.bridge.ip4_address # management_interface.ip_address
ws_ns = self._router.add_interface('websocketclient', address = ip,
port = ns.control_port,
@@ -1252,7 +1275,13 @@ class ResourceManager(metaclass=Singleton):
state = resource._state.state
self.log(resource, 'Current state is {}'.format(state))
- if state == ResourceState.UNINITIALIZED:
+ if state == ResourceState.ERROR:
+ e = resource._state.change_value
+ print("------")
+ import traceback; traceback.print_tb(e.__traceback__)
+ log.error('Resource: {} - Exception: {}'.format(pfx, e))
+ import os; os._exit(1)
+ elif state == ResourceState.UNINITIALIZED:
pending_state = ResourceState.PENDING_DEPS
elif state == ResourceState.DEPS_OK:
pending_state = ResourceState.PENDING_INIT
@@ -1379,24 +1408,26 @@ class ResourceManager(metaclass=Singleton):
# with container.execute(), not container.get()
log.warning('LXD Fix (not found). Reset resource')
new_state = ResourceState.INITIALIZED
+ resource._state.change_success = True
elif ENABLE_LXD_WORKAROUND and isinstance(e, LXDAPIException):
# "not found" is the normal exception when the container
# does not exists. anyways the bug should only occur
# with container.execute(), not container.get()
log.warning('LXD Fix (API error). Reset resource')
new_state = ResourceState.INITIALIZED
+ resource._state.change_success = True
elif isinstance(e, ResourceNotFound):
# The resource does not exist
self.log(resource, S_GET_DONE.format(
resource._state.change_value))
new_state = ResourceState.GET_DONE
resource._state.change_value = None
+ resource._state.change_success = True
else:
e = resource._state.change_value
log.error('Cannot get resource state {} : {}'.format(
resource.get_uuid(), e))
new_state = ResourceState.ERROR
- resource._state.change_success = True
elif pending_state == ResourceState.PENDING_KEYS:
if resource._state.change_success == True:
@@ -1442,10 +1473,7 @@ class ResourceManager(metaclass=Singleton):
resource._state.change_success = True
else:
self.log(resource, 'CREATE failed: {}'.format(e))
- e = resource._state.change_value
- import traceback; traceback.print_tb(e.__traceback__)
- log.error('Failed with exception {}'.format(e))
- import os; os._exit(1)
+ new_state = ResourceState.ERROR
elif pending_state == ResourceState.PENDING_UPDATE:
if resource._state.change_success == True:
@@ -1462,11 +1490,8 @@ class ResourceManager(metaclass=Singleton):
resource._state.change_success = True
resource._state.write_lock.release()
else:
- e = resource._state.change_value
resource._state.write_lock.release()
- import traceback; traceback.print_tb(e.__traceback__)
- log.error('Failed with exception {}'.format(e))
- import os; os._exit(1)
+ new_state = ResourceState.ERROR
elif pending_state == ResourceState.PENDING_DELETE:
raise NotImplementedError
@@ -1475,4 +1500,3 @@ class ResourceManager(metaclass=Singleton):
raise RuntimeError
await self._set_resource_state(resource, new_state)
-
diff --git a/vicn/core/sa_collections.py b/vicn/core/sa_collections.py
index e627caa5..a4a24f85 100644
--- a/vicn/core/sa_collections.py
+++ b/vicn/core/sa_collections.py
@@ -12,8 +12,9 @@
import logging
-from vicn.core.sa_compat import py2k
from vicn.core.exception import VICNListException
+from vicn.core.sa_compat import py2k
+from vicn.core.state import UUID
log = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ def _list_decorators():
try:
item = self._attribute.do_list_add(self._instance, item)
fn(self, item)
- except VICNListException as e:
+ except VICNListException as e:
pass
_tidy(append)
return append
@@ -121,7 +122,7 @@ def _list_decorators():
try:
self._attribute.do_list_remove(self._instance, item)
except : has_except = True
- if not has_except:
+ if not has_except:
fn(self, index)
_tidy(__delitem__)
return __delitem__
@@ -180,7 +181,7 @@ def _list_decorators():
self._attribute.do_list_remove(self._instance, item)
item = fn(self, index)
return item
- except : return None
+ except : return None
_tidy(pop)
return pop
@@ -230,13 +231,27 @@ def _instrument_list(cls):
# inspired by sqlalchemy
for method, decorator in _list_decorators().items():
fn = getattr(cls, method, None)
- if fn:
+ if fn:
#if (fn and method not in methods and
# not hasattr(fn, '_sa_instrumented')):
setattr(cls, method, decorator(fn))
class InstrumentedList(list):
+ @classmethod
+ def from_list(cls, value, instance, attribute):
+ lst = list()
+ if value:
+ for x in value:
+ if isinstance(x, UUID):
+ x = instance.from_uuid(x)
+ lst.append(x)
+ # Having a class method is important for inheritance
+ value = cls(lst)
+ value._attribute = attribute
+ value._instance = instance
+ return value
+
def __contains__(self, key):
from vicn.core.resource import Resource
if isinstance(key, Resource):
diff --git a/vicn/core/state.py b/vicn/core/state.py
index bb108b2b..81876790 100644
--- a/vicn/core/state.py
+++ b/vicn/core/state.py
@@ -75,7 +75,7 @@ class UUID:
random identifier of length UUID_LEN. Components of the UUID are
separated by UUID_SEP.
"""
- uuid = ''.join(random.choice(string.ascii_uppercase + string.digits)
+ uuid = ''.join(random.choice(string.ascii_uppercase + string.digits)
for _ in range(UUID_LEN))
if name:
uuid = name # + UUID_SEP + uuid
@@ -105,7 +105,7 @@ class PendingValue:
if action == Operations.SET:
self.value = value
- self.operations = [(Operations.SET, value)]
+ self.operations = [(Operations.SET, value)]
elif action == Operations.LIST_CLEAR:
self.value = list()
self.operations = [(Operations.LIST_CLEAR, None)]
@@ -136,9 +136,8 @@ class InstanceState:
# LIST set add remove clear
self.dirty = dict()
-
# Initialize resource state
- self.lock = asyncio.Lock()
+ self.lock = asyncio.Lock()
self.write_lock = asyncio.Lock()
self.state = ResourceState.UNINITIALIZED
self.clean = asyncio.Event()
@@ -161,7 +160,7 @@ class InstanceState:
self.attr_log = dict()
# Initialize attribute state
for attribute in instance.iter_attributes():
- self.attr_lock[attribute.name] = asyncio.Lock()
+ self.attr_lock[attribute.name] = asyncio.Lock()
self.attr_init[attribute.name] = asyncio.Event()
self.attr_clean[attribute.name] = asyncio.Event()
self.attr_state[attribute.name] = AttributeState.UNINITIALIZED
diff --git a/vicn/core/task.py b/vicn/core/task.py
index 53321972..8346c65e 100644
--- a/vicn/core/task.py
+++ b/vicn/core/task.py
@@ -219,6 +219,14 @@ class PythonTask(Task):
fut = loop.run_in_executor(None, partial)
fut.add_done_callback(self._done_callback)
+ def execute_blocking(self, *args, **kwargs):
+ all_args = self._args + args
+ all_kwargs = dict()
+ all_kwargs.update(self._kwargs)
+ all_kwargs.update(kwargs)
+
+ return self._func(*all_args, **all_kwargs)
+
def __repr__(self):
s = _get_func_desc(self._func)
return '<Task[py] {}>'.format(s) if s else '<Task[py]>'
diff --git a/vicn/resource/central.py b/vicn/resource/central.py
index 1013d1a5..09b24184 100644
--- a/vicn/resource/central.py
+++ b/vicn/resource/central.py
@@ -22,13 +22,14 @@ import os
from netmodel.model.type import String
from netmodel.util.misc import pairwise
-from vicn.core.attribute import Attribute
+from vicn.core.attribute import Attribute, Reference
from vicn.core.exception import ResourceNotFound
from vicn.core.resource import Resource
from vicn.core.task import async_task, inline_task
from vicn.core.task import EmptyTask, BashTask
from vicn.resource.channel import Channel
from vicn.resource.ip.route import IPRoute
+from vicn.resource.group import Group
from vicn.resource.icn.forwarder import Forwarder
from vicn.resource.icn.face import L2Face, L4Face, FaceProtocol
from vicn.resource.icn.producer import Producer
@@ -148,60 +149,63 @@ MAP_ROUTING_STRATEGY = {
# L2 and L4/ICN graphs
#------------------------------------------------------------------------------
-def _get_l2_graph(manager, with_managed = False):
+def _get_l2_graph(groups, with_managed = False):
G = nx.Graph()
- for node in manager.by_type(Node):
- G.add_node(node._state.uuid)
-
- for channel in manager.by_type(Channel):
- if channel.has_type('emulatedchannel'):
- src = channel._ap_if
- for dst in channel._sta_ifs.values():
- if not with_managed and (not src.managed or not dst.managed):
- continue
- if G.has_edge(src.node._state.uuid, dst.node._state.uuid):
- continue
-
- map_node_interface = { src.node._state.uuid : src._state.uuid,
- dst.node._state.uuid: dst._state.uuid}
- G.add_edge(src.node._state.uuid, dst.node._state.uuid,
- map_node_interface = map_node_interface)
- else:
- # This is for a normal Channel
- for src_it in range(0,len(channel.interfaces)):
- src = channel.interfaces[src_it]
-
- # Iterate over the remaining interface to create all the
- # possible combination
- for dst_it in range(src_it+1,len(channel.interfaces)):
- dst = channel.interfaces[dst_it]
-
- if not with_managed and (not src.managed or
- not dst.managed):
+# for node in manager.by_type(Node):
+# G.add_node(node._state.uuid)
+
+ for group in groups:
+ for channel in group.iter_by_type_str('channel'):
+ if channel.has_type('emulatedchannel'):
+ src = channel._ap_if
+ for dst in channel._sta_ifs.values():
+ if not with_managed and (not src.managed or not dst.managed):
continue
if G.has_edge(src.node._state.uuid, dst.node._state.uuid):
continue
- map_node_interface = {
- src.node._state.uuid : src._state.uuid,
+
+ map_node_interface = { src.node._state.uuid : src._state.uuid,
dst.node._state.uuid: dst._state.uuid}
G.add_edge(src.node._state.uuid, dst.node._state.uuid,
map_node_interface = map_node_interface)
+ else:
+ # This is for a normal Channel
+ for src_it in range(0, len(channel.interfaces)):
+ src = channel.interfaces[src_it]
+
+ # Iterate over the remaining interface to create all the
+ # possible combination
+ for dst_it in range(src_it+1,len(channel.interfaces)):
+ dst = channel.interfaces[dst_it]
+
+ if not with_managed and (not src.managed or
+ not dst.managed):
+ continue
+ if G.has_edge(src.node._state.uuid, dst.node._state.uuid):
+ continue
+ map_node_interface = {
+ src.node._state.uuid : src._state.uuid,
+ dst.node._state.uuid: dst._state.uuid}
+ G.add_edge(src.node._state.uuid, dst.node._state.uuid,
+ map_node_interface = map_node_interface)
return G
-def _get_icn_graph(manager):
+def _get_icn_graph(manager, groups):
G = nx.Graph()
- for forwarder in manager.by_type(Forwarder):
- node = forwarder.node
- G.add_node(node._state.uuid)
- for face in forwarder.faces:
- other_face = manager.by_uuid(face._internal_data['sibling_face'])
- other_node = other_face.node
- if G.has_edge(node._state.uuid, other_node._state.uuid):
- continue
- map_node_face = { node._state.uuid: face._state.uuid,
- other_node._state.uuid: other_face._state.uuid }
- G.add_edge(node._state.uuid, other_node._state.uuid,
- map_node_face = map_node_face)
+ for group in groups:
+ # It's safer to iterate on node which we know are in the right groups,
+ # while it might not be the case for the forwarders...
+ for node in group.iter_by_type_str('node'):
+ G.add_node(node._state.uuid)
+ for face in node.forwarder.faces:
+ other_face = manager.by_uuid(face._internal_data['sibling_face'])
+ other_node = other_face.node
+ if G.has_edge(node._state.uuid, other_node._state.uuid):
+ continue
+ map_node_face = { node._state.uuid: face._state.uuid,
+ other_node._state.uuid: other_face._state.uuid }
+ G.add_edge(node._state.uuid, other_node._state.uuid,
+ map_node_face = map_node_face)
return G
@@ -241,14 +245,18 @@ class IPRoutes(Resource):
def _get_ip_origins(self):
origins = dict()
- for node in self._state.manager.by_type(Node):
- node_uuid = node._state.uuid
- if not node_uuid in origins:
- origins[node_uuid] = list()
- for interface in node.interfaces:
- origins[node_uuid].append(interface.ip4_address)
- if interface.ip6_address: #Control interfaces have no v6 address
- origins[node_uuid].append(interface.ip6_address)
+ for group in self.groups:
+ for node in group.iter_by_type_str('node'):
+ node_uuid = node._state.uuid
+ if not node_uuid in origins:
+ origins[node_uuid] = list()
+ for interface in node.interfaces:
+ # XXX temp fix (WouldBlock)
+ try:
+ origins[node_uuid].append(interface.ip4_address)
+ if interface.ip6_address: #Control interfaces have no v6 address
+ origins[node_uuid].append(interface.ip6_address)
+ except: pass
return origins
def _get_ip_routes(self):
@@ -257,7 +265,7 @@ class IPRoutes(Resource):
strategy = MAP_ROUTING_STRATEGY.get(self.routing_strategy)
- G = _get_l2_graph(self._state.manager)
+ G = _get_l2_graph(self.groups)
origins = self._get_ip_origins()
# node -> list(origins for which we have routes)
@@ -294,7 +302,7 @@ class IPRoutes(Resource):
if prefix == next_hop_ingress_ip:
# Direct route on src_node.name :
# route add [prefix] dev [next_hop_interface_.device_name]
- route4 = IPRoute(node = src_node,
+ route = IPRoute(node = src_node,
managed = False,
owner = self,
ip_address = prefix,
@@ -334,7 +342,7 @@ class IPRoutes(Resource):
IP routing strategy : direct routes only
"""
routes = list()
- G = _get_l2_graph(self._state.manager)
+ G = _get_l2_graph(self.groups)
for src_node_uuid, dst_node_uuid, data in G.edges_iter(data = True):
src_node = self._state.manager.by_uuid(src_node_uuid)
dst_node = self._state.manager.by_uuid(dst_node_uuid)
@@ -424,7 +432,7 @@ class ICNFaces(Resource):
protocol = FaceProtocol.from_string(self.protocol_name)
faces = list()
- G = _get_l2_graph(self._state.manager)
+ G = _get_l2_graph(self.groups)
for src_node_uuid, dst_node_uuid, data in G.edges_iter(data = True):
src_node = self._state.manager.by_uuid(src_node_uuid)
dst_node = self._state.manager.by_uuid(dst_node_uuid)
@@ -436,14 +444,16 @@ class ICNFaces(Resource):
log.debug('{} -> {} ({} -> {})'.format(src_node_uuid,
dst_node_uuid, src.device_name, dst.device_name))
+ # XXX This should be moved to the various faces, that register to a
+ # factory
if protocol == FaceProtocol.ether:
src_face = L2Face(node = src_node,
- owner = self,
+ owner = self,
protocol = protocol,
src_nic = src,
dst_mac = dst.mac_address)
dst_face = L2Face(node = dst_node,
- owner = self,
+ owner = self,
protocol = protocol,
src_nic = dst,
dst_mac = src.mac_address)
@@ -451,14 +461,14 @@ class ICNFaces(Resource):
elif protocol in (FaceProtocol.tcp4, FaceProtocol.tcp6,
FaceProtocol.udp4, FaceProtocol.udp6):
src_face = L4Face(node = src_node,
- owner = self,
+ owner = self,
protocol = protocol,
src_ip = src.ip4_address,
dst_ip = dst.ip4_address,
src_port = TMP_DEFAULT_PORT,
dst_port = TMP_DEFAULT_PORT)
dst_face = L4Face(node = dst_node,
- owner = self,
+ owner = self,
protocol = protocol,
src_ip = dst.ip4_address,
dst_ip = src.ip4_address,
@@ -510,17 +520,18 @@ class ICNRoutes(Resource):
def _get_prefix_origins(self):
origins = dict()
- for producer in self._state.manager.by_type(Producer):
- node_uuid = producer.node._state.uuid
- if not node_uuid in origins:
- origins[node_uuid] = list()
- origins[node_uuid].extend(producer.prefixes)
+ for group in self.groups:
+ for producer in group.iter_by_type_str('producer'):
+ node_uuid = producer.node._state.uuid
+ if not node_uuid in origins:
+ origins[node_uuid] = list()
+ origins[node_uuid].extend(producer.prefixes)
return origins
def _get_icn_routes(self):
strategy = MAP_ROUTING_STRATEGY.get(self.routing_strategy)
- G = _get_icn_graph(self._state.manager)
+ G = _get_icn_graph(self._state.manager, self.groups)
origins = self._get_prefix_origins()
routes = list()
@@ -571,104 +582,6 @@ class DnsServerEntry(Resource):
#------------------------------------------------------------------------------
-class ContainerSetup(Resource):
- """
- Resource: ContainerSetup
-
- Setup of container networking
-
- Todo:
- - This should be merged into the LxcContainer resource
- """
-
- container = Attribute(LxcContainer)
-
- #--------------------------------------------------------------------------
- # Resource lifecycle
- #--------------------------------------------------------------------------
-
- def __subresources__(self):
-
- dns_server_entry = DnsServerEntry(node = self.container,
- owner = self,
- ip_address = self.container.node.bridge.ip4_address,
- interface_name = self.container.host_interface.device_name)
-
- return dns_server_entry
-
- @inline_task
- def __get__(self):
- raise ResourceNotFound
-
- def __create__(self):
- #If no IP has been given on the host interface (e.g., through DHCP)
- #We need to assign one
- if not self.container.host_interface.ip4_address:
- # a) get the IP
- assign=self._state.manager.by_type(Ipv4Assignment)[0]
- ip4_addr = assign.get_control_address(self.container.host_interface)
- self.container.host_interface.ip4_address = ip4_addr
-
-
- # a) routes: host -> container
- # . container interfaces
- # . container host (main) interface
- # route add -host {ip_address} dev {bridge_name}
- route = IPRoute(node = self.container.node,
- managed = False,
- owner = self,
- ip_address = ip4_addr,
- interface = self.container.node.bridge)
- route.node.routing_table.routes << route
-
- # b) route: container -> host
- # route add {ip_gateway} dev {interface_name}
- # route add default gw {ip_gateway} dev {interface_name}
- route = IPRoute(node = self.container,
- owner = self,
- managed = False,
- ip_address = self.container.node.bridge.ip4_address,
- interface = self.container.host_interface)
- route.node.routing_table.routes << route
- route_gw = IPRoute(node = self.container,
- managed = False,
- owner = self,
- ip_address = 'default',
- interface = self.container.host_interface,
- gateway = self.container.node.bridge.ip4_address)
- route_gw.node.routing_table.routes << route_gw
-
-
- return BashTask(self.container.node, CMD_IP_FORWARD)
-
-#------------------------------------------------------------------------------
-
-class ContainersSetup(Resource):
- """
- Resource: ContainersSetup
-
- Setup of LxcContainers (main resource)
-
- Todo:
- - This should be merged into the LxcContainer resource
- """
-
- #--------------------------------------------------------------------------
- # Resource lifecycle
- #--------------------------------------------------------------------------
-
- def __subresources__(self):
- containers = self._state.manager.by_type(LxcContainer)
- if len(containers) == 0:
- return None
-
- container_resources = [ContainerSetup(owner = self, container = c)
- for c in containers]
-
- return Resource.__concurrent__(*container_resources)
-
-#------------------------------------------------------------------------------
-
class CentralIP(Resource):
"""
Resource: CentralIP
@@ -678,32 +591,31 @@ class CentralIP(Resource):
ip_routing_strategy = Attribute(String, description = 'IP routing strategy',
default = 'pair') # spt, pair
- ip6_data_prefix = Attribute(String, description="Prefix for IPv6 forwarding", mandatory=True)
- ip4_data_prefix = Attribute(String, description="Prefix for IPv4 forwarding", mandatory=True)
- ip4_control_prefix = Attribute(String, description="Prefix for IPv4 control", mandatory=True)
+ ip6_data_prefix = Attribute(String, description="Prefix for IPv6 forwarding",
+ mandatory = True)
+ ip4_data_prefix = Attribute(String, description="Prefix for IPv4 forwarding",
+ mandatory = True)
#--------------------------------------------------------------------------
# Resource lifecycle
#--------------------------------------------------------------------------
- #def __after_init__(self):
- # return ('Node', 'Channel', 'Interface')
+ def __after_init__(self):
+ return ('Node', 'Channel', 'Interface')
+
+ def __after__(self):
+ return ('EmulatedChannel')
def __subresources__(self):
- ip4_assign = Ipv4Assignment(prefix=self.ip4_data_prefix,
- control_prefix=self.ip4_control_prefix)
- ip6_assign = Ipv6Assignment(prefix=self.ip6_data_prefix)
- containers_setup = ContainersSetup(owner=self)
+ ip4_assign = Ipv4Assignment(prefix = self.ip4_data_prefix,
+ groups = Reference(self, 'groups'))
+ ip6_assign = Ipv6Assignment(prefix = self.ip6_data_prefix,
+ groups = Reference(self, 'groups'))
ip_routes = IPRoutes(owner = self,
+ groups = Reference(self, 'groups'),
routing_strategy = self.ip_routing_strategy)
- return (ip4_assign | ip6_assign) > (ip_routes | containers_setup)
-
- @inline_task
- def __get__(self):
- raise ResourceNotFound
-
- __delete__ = None
+ return (ip4_assign | ip6_assign) > ip_routes
#------------------------------------------------------------------------------
@@ -734,9 +646,11 @@ class CentralICN(Resource):
return ('CentralIP',)
def __subresources__(self):
- icn_faces = ICNFaces(owner = self, protocol_name = self.face_protocol)
+ icn_faces = ICNFaces(owner = self, protocol_name = self.face_protocol,
+ groups = Reference(self, 'groups'))
icn_routes = ICNRoutes(owner = self,
- routing_strategy = self.icn_routing_strategy)
+ routing_strategy = self.icn_routing_strategy,
+ groups = Reference(self, 'groups'))
return icn_faces > icn_routes
@inline_task
diff --git a/vicn/resource/channel.py b/vicn/resource/channel.py
index d91bebcf..4576e0e7 100644
--- a/vicn/resource/channel.py
+++ b/vicn/resource/channel.py
@@ -22,8 +22,6 @@ from vicn.core.attribute import Attribute
from vicn.core.task import EmptyTask
from vicn.resource.ip_assignment import Ipv6Assignment, Ipv4Assignment
-from math import log, ceil
-
class Channel(Resource):
"""
Resource: Channel
@@ -49,26 +47,3 @@ class Channel(Resource):
ret = "{:03}".format(len(self.interfaces))
ret = ret + ''.join(sorted(map(lambda x : x.node.name, self.interfaces)))
return ret
-
- def __create__(self):
- interfaces = sorted(self.interfaces, key = lambda x : x.device_name)
- if interfaces:
- #IPv6
- central6 = self._state.manager.by_type(Ipv6Assignment)[0]
- prefix6_size = min(64, 128 - ceil(log(len(self.interfaces), 2)))
- prefix6 = iter(central6.get_prefix(self, prefix6_size))
-
- #IPv4
- central4 = self._state.manager.by_type(Ipv4Assignment)[0]
- prefix4_size = 32 - ceil(log(len(self.interfaces), 2))
- prefix4 = iter(central4.get_prefix(self, prefix4_size))
-
- for interface in interfaces:
- try:
- interface.ip4_address = next(prefix4)
- except StopIteration as e:
- import pdb; pdb.set_trace()
- interface.ip6_address = next(prefix6)
- interface.ip6_prefix = prefix6_size
-
- return EmptyTask()
diff --git a/vicn/resource/group.py b/vicn/resource/group.py
new file mode 100644
index 00000000..1557c42e
--- /dev/null
+++ b/vicn/resource/group.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from vicn.core.resource import Resource
+from vicn.core.attribute import Attribute, Multiplicity
+
+class Group(Resource):
+ resources = Attribute(Resource, description = 'Resources belonging to the group',
+ multiplicity = Multiplicity.ManyToMany,
+ default = [],
+ reverse_name = 'groups',
+ reverse_description = 'Groups to which the resource belongs')
+
+ def iter_by_type(self, type):
+ for r in self.resources:
+ if isinstance(r, type):
+ yield r
+
+ def iter_by_type_str(self, typestr):
+ cls = self._state.manager._available.get(typestr.lower())
+ if not cls:
+ return list()
+ return self.iter_by_type(cls)
diff --git a/vicn/resource/icn/ndnpingserver.py b/vicn/resource/icn/ndnpingserver.py
index da13f59b..f9cfa7cc 100644
--- a/vicn/resource/icn/ndnpingserver.py
+++ b/vicn/resource/icn/ndnpingserver.py
@@ -55,7 +55,7 @@ class NDNPingServerBase(Producer):
node = Attribute(requirements = [
Requirement("forwarder",
- capabilities = set(['ICN_SUITE_CCNX_1_0'])) ])
+ capabilities = set(['ICN_SUITE_NDN_1_0'])) ])
__package_names__ = ['ndnping']
diff --git a/vicn/resource/icn/webserver.py b/vicn/resource/icn/webserver.py
index 8b8e2ef3..71e9f200 100644
--- a/vicn/resource/icn/webserver.py
+++ b/vicn/resource/icn/webserver.py
@@ -25,5 +25,5 @@ class WebServer(Producer):
CCNX Webserver
"""
- __package_names__ = ['webserver-ccnx']
- __service_name__ = 'webserver-ccnx'
+ __package_names__ = ['http-server']
+ __service_name__ = 'http-server'
diff --git a/vicn/resource/ip_assignment.py b/vicn/resource/ip_assignment.py
index 7553f4ff..62a32389 100644
--- a/vicn/resource/ip_assignment.py
+++ b/vicn/resource/ip_assignment.py
@@ -16,6 +16,8 @@
# limitations under the License.
#
+import math
+
from vicn.core.resource import Resource
from netmodel.model.type import String
from vicn.core.attribute import Attribute
@@ -60,9 +62,42 @@ class IpAssignment(Resource):
self._assigned_addresses[obj] = ret
return ret
+ @inline_task
+ def __get__(self):
+ raise ResourceNotFound
+
+ @inline_task
+ def __create__(self):
+ # XXX code from Channel.__create__, until Events are properly implemented.
+ # Iterate on channels for allocate IP addresses
+ for group in self.groups:
+ for channel in group.iter_by_type_str('channel'):
+ interfaces = sorted(channel.interfaces, key = lambda x : x.device_name)
+ if not interfaces:
+ continue
+
+ min_prefix_size = math.ceil(math.log(len(channel.interfaces), 2))
+ prefix_size = min(self.DEFAULT_PREFIX_SIZE, self.MAX_PREFIX_SIZE - min_prefix_size)
+ prefix = iter(self.get_prefix(channel, prefix_size))
+
+ for interface in interfaces:
+ ip = next(prefix)
+ print('attribute ip=', ip)
+ setattr(interface, self.ATTR_ADDRESS, ip)
+ setattr(interface, self.ATTR_PREFIX, prefix_size)
+
+ __delete__ = None
+
class Ipv6Assignment(IpAssignment):
PrefixClass = Inet6Prefix
-
+ DEFAULT_PREFIX_SIZE = 64
+ MAX_PREFIX_SIZE = 128
+ ATTR_ADDRESS = 'ip6_address'
+ ATTR_PREFIX = 'ip6_prefix'
class Ipv4Assignment(IpAssignment):
PrefixClass = Inet4Prefix
+ DEFAULT_PREFIX_SIZE = 32
+ MAX_PREFIX_SIZE = 32
+ ATTR_ADDRESS = 'ip4_address'
+ ATTR_PREFIX = 'ip4_prefix'
diff --git a/vicn/resource/linux/application.py b/vicn/resource/linux/application.py
index d2b5139e..ed135da6 100644
--- a/vicn/resource/linux/application.py
+++ b/vicn/resource/linux/application.py
@@ -21,7 +21,6 @@ from vicn.core.resource import Resource, EmptyResource
from vicn.resource.application import Application
from vicn.resource.linux.package_manager import Packages
-
class LinuxApplication(Application):
"""
Resource: Linux Application
diff --git a/vicn/resource/linux/bridge.py b/vicn/resource/linux/bridge.py
index 882f0226..7b5ceed7 100644
--- a/vicn/resource/linux/bridge.py
+++ b/vicn/resource/linux/bridge.py
@@ -46,7 +46,7 @@ class Bridge(Channel, BaseNetDevice):
Requirement('bridge_manager')
])
device_name = Attribute(
- default = DEFAULT_BRIDGE_NAME,
+ default = lambda self: self._state.manager.get('bridge_name'),
mandatory = False)
#--------------------------------------------------------------------------
diff --git a/vicn/resource/linux/dnsmasq.py b/vicn/resource/linux/dnsmasq.py
index e18f750f..b5aa8053 100644
--- a/vicn/resource/linux/dnsmasq.py
+++ b/vicn/resource/linux/dnsmasq.py
@@ -42,7 +42,6 @@ TPL_CONF='''
interface=$interface
dhcp-range=$dhcp_range
-dhcp-host=00:0e:c6:81:79:01,192.168.128.200,12h
#server=$server
$flags
@@ -60,12 +59,12 @@ class DnsMasq(Service, DnsServer):
__package_names__ = ['dnsmasq']
__service_name__ = 'dnsmasq'
- interface = Attribute(Interface,
+ interface = Attribute(Interface,
description = 'Interface on which to listen')
lease_interval = Attribute(String,
default = '12h')
server = Attribute(String)
- dhcp_authoritative = Attribute(Bool,
+ dhcp_authoritative = Attribute(Bool,
description = 'Flag: DHCP authoritative',
default = True)
log_queries = Attribute(Bool, description = 'Flag: log DNS queries',
@@ -80,10 +79,7 @@ class DnsMasq(Service, DnsServer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if not self.interface:
- if self.node.bridge:
- self.interface = self.node.bridge
- else:
- self.interface = self.node.host_interface
+ raise Exception("Cannot initialize bridge without interface")
def __subresources__(self):
# Overwrite configuration file
diff --git a/vicn/resource/linux/net_device.py b/vicn/resource/linux/net_device.py
index 1ce7e4d5..e40256ea 100644
--- a/vicn/resource/linux/net_device.py
+++ b/vicn/resource/linux/net_device.py
@@ -22,14 +22,14 @@ import math
import random
import string
-from netmodel.model.type import Integer, String, Bool
-from vicn.core.address_mgr import AddressManager
-from vicn.core.attribute import Attribute
-from vicn.core.exception import ResourceNotFound
-from vicn.core.resource import BaseResource
-from vicn.core.task import BashTask, task, EmptyTask
-from vicn.resource.application import Application
-from vicn.resource.interface import Interface
+from netmodel.model.type import Integer, String, Bool
+from vicn.core.address_mgr import AddressManager
+from vicn.core.attribute import Attribute
+from vicn.core.exception import ResourceNotFound
+from vicn.core.resource import BaseResource
+from vicn.core.task import BashTask, task, EmptyTask
+from vicn.resource.linux.application import LinuxApplication as Application
+from vicn.resource.interface import Interface
# parse_ip_addr inspired from:
# From: https://github.com/ohmu/poni/blob/master/poni/cloud_libvirt.py
diff --git a/vicn/resource/linux/package_manager.py b/vicn/resource/linux/package_manager.py
index 86b7057a..eaf83e17 100644
--- a/vicn/resource/linux/package_manager.py
+++ b/vicn/resource/linux/package_manager.py
@@ -19,7 +19,7 @@
import asyncio
import logging
-from netmodel.model.type import String
+from netmodel.model.type import String, Bool
from vicn.core.attribute import Attribute, Multiplicity
from vicn.core.exception import ResourceNotFound
from vicn.core.requirement import Requirement
@@ -63,7 +63,7 @@ class PackageManager(Resource):
Resource: PackageManager
APT package management wrapper.
-
+
Todo:
- We assume a package manager is always installed on every machine.
- Currently, we limit ourselves to debian/ubuntu, and voluntarily don't
@@ -79,6 +79,9 @@ class PackageManager(Resource):
reverse_auto = True,
mandatory = True,
multiplicity = Multiplicity.OneToOne)
+ trusted = Attribute(Bool,
+ description="Force repository trust",
+ default=False)
#--------------------------------------------------------------------------
# Constructor and Accessors
@@ -94,34 +97,28 @@ class PackageManager(Resource):
#--------------------------------------------------------------------------
def __after__(self):
- if self.node.__class__.__name__ == 'Physical':
- # UGLY : This blocking code is currently needed
- task = self.node.host_interface._get_ip4_address()
- ip_dict = task.execute_blocking()
- self.node.host_interface.ip4_address = ip_dict['ip4_address']
- return ('Repository',)
- else:
- return ('Repository', 'CentralIP', 'RoutingTable')
+ return ('Repository',)
@inline_task
def __get__(self):
raise ResourceNotFound
- def __create__(self):
+ #---------------------------------------------------------------------------
+ # Methods
+ #---------------------------------------------------------------------------
+
+ def __method_setup_repositories__(self):
repos = EmptyTask()
for repository in self._state.manager.by_type_str('Repository'):
deb_source = self._get_deb_source(repository)
path = self._get_path(repository)
- repo = BashTask(self.node, CMD_SETUP_REPO,
+ # XXX There is no need to setup a repo if there is no package to install
+ repo = BashTask(self.node, CMD_SETUP_REPO,
{'deb_source': deb_source, 'path': path})
repos = repos | repo
- return repos
+ return repos
- #---------------------------------------------------------------------------
- # Methods
- #---------------------------------------------------------------------------
-
def __method_update__(self):
kill = BashTask(self.node, CMD_APT_GET_KILL, {'node': self.node.name},
lock = self.apt_lock)
@@ -139,13 +136,12 @@ class PackageManager(Resource):
else:
update = EmptyTask()
- return (kill > dpkg_configure_a) > update
+ return (self.__method_setup_repositories__() > (kill > dpkg_configure_a)) > update
def __method_install__(self, package_name):
- update = self.__method_update__()
install = BashTask(self.node, CMD_PKG_INSTALL, {'package_name':
package_name}, lock = self.apt_lock)
- return update > install
+ return self.__method_update__() > install
#---------------------------------------------------------------------------
# Internal methods
@@ -158,10 +154,20 @@ class PackageManager(Resource):
return '/etc/apt/sources.list.d/{}.list'.format(repository.repo_name)
def _get_deb_source(self, repository):
- path = repository.node.host_interface.ip4_address + '/'
+ protocol = 'https' if repository.ssl else 'http'
+ path = repository.node.hostname + '/'
if repository.directory:
path += repository.directory + '/'
- return 'deb http://{} {}/'.format(path, self.node.dist)
+ trusted = '[trusted=yes] ' if self.trusted else ''
+ if repository.sections:
+ sections = ' {}'.format(' '.join(repository.sections))
+ else:
+ sections = ''
+ if '$DISTRIBUTION' in path:
+ path = path.replace('$DISTRIBUTION', self.node.dist)
+ return 'deb {}{}://{} ./{}'.format(trusted, protocol, path, sections)
+ else:
+ return 'deb {}{}://{} {}{}'.format(trusted, protocol, path, self.node.dist, sections)
#------------------------------------------------------------------------------
@@ -173,7 +179,7 @@ class Package(Resource):
"""
package_name = Attribute(String, mandatory = True)
- node = Attribute(Node,
+ node = Attribute(Node,
mandatory = True,
requirements=[
Requirement('package_manager')
@@ -208,7 +214,7 @@ class Packages(Resource):
since package_names are static for a resource, this is not a problem here.
"""
names = Attribute(String, multiplicity = Multiplicity.OneToMany)
- node = Attribute(Node,
+ node = Attribute(Node,
mandatory = True,
requirements=[
Requirement('package_manager')
@@ -229,4 +235,3 @@ class Packages(Resource):
return Resource.__concurrent__(*packages)
else:
return None
-
diff --git a/vicn/resource/linux/repository.py b/vicn/resource/linux/repository.py
index f3e70565..cd740d38 100644
--- a/vicn/resource/linux/repository.py
+++ b/vicn/resource/linux/repository.py
@@ -16,7 +16,7 @@
# limitations under the License.
#
-from netmodel.model.type import String
+from netmodel.model.type import String, Bool
from vicn.core.attribute import Attribute, Multiplicity
from vicn.resource.application import Application
@@ -35,7 +35,12 @@ class Repository(Application):
default = 'vicn')
directory = Attribute(String, description = 'Directory holding packages',
default = '')
+ sections = Attribute(String, description = 'Sections',
+ multiplicity = Multiplicity.OneToMany,
+ default = [])
distributions = Attribute(String,
description = 'List of distributions served by this repository',
multiplicity = Multiplicity.ManyToMany,
default = ['sid', 'trusty', 'xenial'])
+ ssl = Attribute(Bool, description = 'Use SSL (https) for repository',
+ default = True)
diff --git a/vicn/resource/lxd/lxc_container.py b/vicn/resource/lxd/lxc_container.py
index 9daaffb7..5670d1a2 100644
--- a/vicn/resource/lxd/lxc_container.py
+++ b/vicn/resource/lxd/lxc_container.py
@@ -18,16 +18,14 @@
import logging
import shlex
-import time
-# Suppress logging from pylxd dependency on ws4py
+# Suppress logging from pylxd dependency on ws4py
# (this needs to be included before pylxd)
from ws4py import configure_logger
configure_logger(level=logging.ERROR)
import pylxd
from netmodel.model.type import String, Integer, Bool, Self
-from vicn.core.address_mgr import AddressManager
from vicn.core.attribute import Attribute, Reference, Multiplicity
from vicn.core.commands import ReturnValue
from vicn.core.exception import ResourceNotFound
@@ -37,12 +35,10 @@ from vicn.core.task import task, inline_task, BashTask, EmptyTas
from vicn.resource.linux.net_device import NetDevice
from vicn.resource.node import Node
from vicn.resource.vpp.scripts import APPARMOR_VPP_PROFILE
+from vicn.resource.lxd.lxd_profile import LXD_PROFILE_DEFAULT_IFNAME
log = logging.getLogger(__name__)
-# Default name of VICN management/monitoring interface
-DEFAULT_LXC_NETDEVICE = 'eth0'
-
# Default remote server (pull mode only)
DEFAULT_SOURCE_URL = 'https://cloud-images.ubuntu.com/releases/'
@@ -56,8 +52,10 @@ CMD_UNSET_IP6_FWD = 'sysctl -w net.ipv6.conf.all.forwarding=0'
CMD_SET_IP6_FWD = 'sysctl -w net.ipv6.conf.all.forwarding=1'
CMD_GET_IP6_FWD = 'sysctl -n net.ipv6.conf.all.forwarding'
+CMD_NETWORK_DHCP='dhclient {container.management_interface.device_name}'
+
# Type: ContainerName
-ContainerName = String(max_size = 64, ascii = True,
+ContainerName = String(max_size = 64, ascii = True,
forbidden = ('/', ',', ':'))
class LxcContainer(Node):
@@ -74,12 +72,12 @@ class LxcContainer(Node):
architecture = Attribute(String, description = 'Architecture',
default = 'x86_64')
- container_name = Attribute(ContainerName,
+ container_name = Attribute(ContainerName,
description = 'Name of the container',
default = Reference(Self, 'name'))
ephemeral = Attribute(Bool, description = 'Ephemeral container flag',
default = False)
- node = Attribute(Node,
+ node = Attribute(Node,
description = 'Node on which the container is running',
mandatory = True,
requirements = [
@@ -91,26 +89,26 @@ class LxcContainer(Node):
Requirement('bridge'),
# A DNS server is required to provide internet connectivity to
# the containers
- Requirement('dns_server'),
+ # Requirement('dns_server'),
])
- profiles = Attribute(String, multiplicity = Multiplicity.OneToMany,
- default = ['default'])
+ profiles = Attribute(String, multiplicity = Multiplicity.OneToMany,
+ default = ['vicn'])
image = Attribute(String, description = 'image', default = None)
is_image = Attribute(Bool, defaut = False)
pid = Attribute(Integer, description = 'PID of the container')
ip6_forwarding = Attribute(Bool, default=True)
- #--------------------------------------------------------------------------
+ #--------------------------------------------------------------------------
# Constructor / Accessors
- #--------------------------------------------------------------------------
+ #--------------------------------------------------------------------------
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._container = None
- #--------------------------------------------------------------------------
+ #--------------------------------------------------------------------------
# Resource lifecycle
- #--------------------------------------------------------------------------
+ #--------------------------------------------------------------------------
@inline_task
def __initialize__(self):
@@ -120,11 +118,11 @@ class LxcContainer(Node):
self.node_with_kernel = Reference(self, 'node')
# We automatically add the management/monitoring interface
- self._host_interface = NetDevice(node = self,
+ self._management_interface = NetDevice(node = self,
owner = self,
monitored = False,
- device_name = DEFAULT_LXC_NETDEVICE)
- self._state.manager.commit_resource(self._host_interface)
+ device_name = LXD_PROFILE_DEFAULT_IFNAME)
+ self._state.manager.commit_resource(self._management_interface)
for iface in self.interfaces:
if iface.get_type() == "dpdkdevice":
@@ -150,7 +148,9 @@ class LxcContainer(Node):
wait_vpp_host = wait_resource_task(self.node.vpp_host)
create = self._create_container()
start = self.__method_start__()
- return wait_vpp_host > (create > start)
+ #XXX Should be an option on the netdevice
+ dhcp_interface = BashTask(self, CMD_NETWORK_DHCP, {'container':self})
+ return (wait_vpp_host > (create > start)) > dhcp_interface
@task
def _create_container(self):
@@ -158,54 +158,52 @@ class LxcContainer(Node):
log.debug('Container description: {}'.format(container))
client = self.node.lxd_hypervisor.client
self._container = client.containers.create(container, wait=True)
- self._container.start(wait = True)
+ #self._container.start(wait = True)
def _get_container_description(self):
# Base configuration
container = {
- 'name' : self.container_name,
+ 'name' : self.container_name,
'architecture' : self.architecture,
- 'ephemeral' : self.ephemeral,
- 'profiles' : ['default'],
+ 'ephemeral' : self.ephemeral,
+ 'profiles' : self.profiles,
'config' : {},
'devices' : {},
}
# DEVICES
- devices = {}
# FIXME Container profile support is provided by setting changes into
# configuration (currently only vpp profile is supported)
for profile in self.profiles:
if profile == 'vpp':
# Set the new apparmor profile. This will be created in VPP
- # application
+ # application
# Mount hugetlbfs in the container.
container['config']['raw.lxc'] = APPARMOR_VPP_PROFILE
container['config']['security.privileged'] = 'true'
for device in self.node.vpp_host.uio_devices:
container['devices'][device] = {
- 'path' : '/dev/{}'.format(device),
+ 'path' : '/dev/{}'.format(device),
'type' : 'unix-char' }
- # NETWORK (not for images)
-
- if not self.is_image:
- container['config']['user.network_mode'] = 'link-local'
- device = {
- 'type' : 'nic',
- 'name' : self.host_interface.device_name,
- 'nictype' : 'bridged',
- 'parent' : self.node.bridge.device_name,
- }
- device['hwaddr'] = AddressManager().get_mac(self)
- prefix = 'veth-{}'.format(self.container_name)
- device['host_name'] = AddressManager().get('device_name', self,
- prefix = prefix, scope = prefix)
-
- container['devices'][device['name']] = device
-
+# # NETWORK (not for images)
+#
+# if not self.is_image:
+# container['config']['user.network_mode'] = 'link-local'
+# device = {
+# 'type' : 'nic',
+# 'name' : self.host_interface.device_name,
+# 'nictype' : 'bridged',
+# 'parent' : self.node.bridge.device_name,
+# }
+# device['hwaddr'] = AddressManager().get_mac(self)
+# prefix = 'veth-{}'.format(self.container_name)
+# device['host_name'] = AddressManager().get('device_name', self,
+# prefix = prefix, scope = prefix)
+#
+# container['devices'][device['name']] = device
# SOURCE
@@ -233,6 +231,7 @@ class LxcContainer(Node):
@task
def __delete__(self):
log.info("Delete container {}".format(self.container_name))
+ import pdb; pdb.set_trace()
self.node.lxd_hypervisor.client.containers.remove(self.name)
#--------------------------------------------------------------------------
@@ -243,7 +242,7 @@ class LxcContainer(Node):
"""
Attribute: pid (getter)
"""
- return BashTask(self.node, CMD_GET_PID, {'container': self},
+ return BashTask(self.node, CMD_GET_PID, {'container': self},
parse = lambda rv: {'pid': rv.stdout.strip()})
#--------------------------------------------------------------------------
diff --git a/vicn/resource/lxd/lxd_hypervisor.py b/vicn/resource/lxd/lxd_hypervisor.py
index 68b7ab28..f9952e4f 100644
--- a/vicn/resource/lxd/lxd_hypervisor.py
+++ b/vicn/resource/lxd/lxd_hypervisor.py
@@ -38,6 +38,7 @@ from vicn.core.task import BashTask, task
from vicn.resource.linux.application import LinuxApplication as Application
from vicn.resource.linux.service import Service
from vicn.resource.linux.certificate import Certificate
+from vicn.resource.lxd.lxd_profile import LxdProfile
# Suppress non-important logging messages from requests and urllib3
logging.getLogger("requests").setLevel(logging.WARNING)
@@ -52,19 +53,20 @@ DEFAULT_KEY_PATH = os.path.expanduser(os.path.join(
'~', '.vicn', 'lxd_client_cert', 'client_key.pem'))
# FIXME hardcoded password for LXD server
-DEFAULT_TRUST_PASSWORD = 'vicn'
+LXD_TRUST_PWD_DEFAULT = 'vicn'
-DEFAULT_LXD_STORAGE = 100 # GB
+LXD_STORAGE_SIZE_DEFAULT = 100 # GB
+LXD_NETWORK_DEFAULT = 'lxdbr-vicn'
+LXD_PROFILE_NAME_DEFAULT = 'vicn'
+ZFS_DEFAULT_POOL_NAME = 'vicn'
# Commands used to interact with the LXD hypervisor
CMD_LXD_CHECK_INIT = 'lsof -i:{lxd.lxd_port}'
CMD_LXD_INIT_BASE = 'lxd init --auto '
-CMD_LXD_INIT='''
-{base}
-lxc profile unset default environment.http_proxy
-lxc profile unset default user.network_mode
-'''
+
+CMD_LXD_NETWORK_GET = 'lxc network list | grep {lxd_hypervisor.network}'
+CMD_LXD_NETWORK_SET = 'lxc network create {lxd_hypervisor.network} || true'
#------------------------------------------------------------------------------
# Subresources
@@ -82,7 +84,7 @@ class LxdInit(Application):
'storage-backend' : self.owner.storage_backend,
'network-port' : self.owner.lxd_port,
'network-address' : '0.0.0.0',
- 'trust-password' : DEFAULT_TRUST_PASSWORD,
+ 'trust-password' : self.owner.trust_password,
}
if self.owner.storage_backend == 'zfs':
@@ -104,8 +106,7 @@ class LxdInit(Application):
# error: Failed to create the ZFS pool: The ZFS modules are not loaded.
# Try running '/sbin/modprobe zfs' as root to load them.
# zfs-dkms in the host
- return BashTask(self.owner.node, CMD_LXD_INIT, {'base': cmd},
- as_root = True)
+ return BashTask(self.owner.node, cmd, as_root = True)
def __delete__(self):
raise NotImplementedError
@@ -134,7 +135,7 @@ class LxdInstallCert(Resource):
client certificate for the LXD daemon.
"""
log.info('Adding certificate on LXD')
- self.owner.client.authenticate(DEFAULT_TRUST_PASSWORD)
+ self.owner.client.authenticate(self.owner.trust_password)
if not self.owner.client.trusted:
raise Exception
@@ -154,9 +155,13 @@ class LxdHypervisor(Service):
default = 'zfs',
choices = ['zfs'])
storage_size = Attribute(Integer, description = 'Storage size',
- default = DEFAULT_LXD_STORAGE) # GB
+ default = LXD_STORAGE_SIZE_DEFAULT) # GB
zfs_pool = Attribute(String, description = 'ZFS pool',
- default='vicn')
+ default=ZFS_DEFAULT_POOL_NAME)
+ network = Attribute(String, description = 'LXD network name',
+ default=LXD_NETWORK_DEFAULT)
+ trust_password = Attribute(String, description = 'Trust password for the LXD server',
+ default=LXD_TRUST_PWD_DEFAULT)
# Just overload attribute with a new reverse
node = Attribute(
@@ -194,8 +199,13 @@ class LxdHypervisor(Service):
owner = self)
lxd_cert_install = LxdInstallCert(certificate = lxd_local_cert,
owner = self)
+ lxd_vicn_profile = LxdProfile(name=LXD_PROFILE_NAME_DEFAULT,
+ node=self.node,
+ description='vICN profile',
+ network=self.network,
+ pool=self.zfs_pool)
- return (lxd_init | lxd_local_cert) > lxd_cert_install
+ return (lxd_init | lxd_local_cert) > (lxd_vicn_profile | lxd_cert_install)
#--------------------------------------------------------------------------
# Private methods
@@ -221,3 +231,10 @@ class LxdHypervisor(Service):
@property
def aliases(self):
return [alias for image in self.images for alias in image.aliases]
+
+ @task
+ def _get_network(self):
+ return None #XXX We assume it's always nothing
+
+ def _set_network(self):
+ return BashTask(self.node, CMD_LXD_NETWORK_SET, {'lxd_hypervisor': self})
diff --git a/vicn/resource/lxd/lxd_profile.py b/vicn/resource/lxd/lxd_profile.py
new file mode 100644
index 00000000..e7afee4a
--- /dev/null
+++ b/vicn/resource/lxd/lxd_profile.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 Cisco and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from vicn.core.resource import Resource
+from netmodel.model.type import String
+from vicn.core.attribute import Attribute, Multiplicity
+from vicn.core.task import BashTask
+from vicn.core.exception import ResourceNotFound
+
+CMD_LXD_PROFILE_CREATE = '''
+lxc profile create {profile.name} description="{profile.description}"
+lxc profile device add {profile.name} root disk pool={profile.pool} path=/
+lxc profile device add {profile.name} {profile.iface_name} nic name={profile.iface_name} nictype=bridged parent={profile.network}
+lxc profile unset {profile.name} environment.http_proxy
+lxc profile unset {profile.name} user.network_mode
+'''
+
+CMD_LXD_PROFILE_GET = 'lxc profile list | grep {profile.name}'
+
+# Default name of VICN management/monitoring interface
+#
+# This should be kept in sync with /etc/network/interfaces in the image file so that dhcp works
+LXD_PROFILE_DEFAULT_IFNAME = 'vicn_mgmt'
+
+class LxdProfile(Resource):
+
+ description = Attribute(String, descr="profile description", mandatory=True)
+ pool = Attribute(String, descr="ZFS pool used by the containers", mandatory=True)
+ network = Attribute(String, description='Network on which to attach', mandatory=True)
+ iface_name = Attribute(String, description='Default interface name',
+ default = LXD_PROFILE_DEFAULT_IFNAME)
+ node = Attribute(Resource, mandatory=True)
+
+ def __get__(self):
+ def parse(rv):
+ if not rv.stdout:
+ raise ResourceNotFound
+ return BashTask(self.node, CMD_LXD_PROFILE_GET, {'profile':self}, parse=parse)
+
+ def __create__(self):
+ return BashTask(self.node, CMD_LXD_PROFILE_CREATE, {'profile':self})
diff --git a/vicn/resource/node.py b/vicn/resource/node.py
index ad519666..c785e32b 100644
--- a/vicn/resource/node.py
+++ b/vicn/resource/node.py
@@ -61,33 +61,17 @@ class Node(Resource):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self._host_interface = None
+ self._management_interface = None
#---------------------------------------------------------------------------
# Public API
#---------------------------------------------------------------------------
@property
- def host_interface(self):
- """
- We assume that any unmanaged interface associated to the host is the
- main host interface. It should thus be declared in the JSON topology.
- We might later perform some kind of auto discovery.
-
- This unmanaged interface is only required to get the device_name:
- - to create Veth (need a parent)
- - to ssh a node, get its ip address (eg for the repo)
- - to avoid loops in type specification
-
- It is used for all nodes to provide network connectivity.
- """
-
- for interface in self.interfaces:
- if not interface.managed or interface.owner is not None:
- return interface
-
- raise Exception('Cannot find host interface for node {}: {}'.format(
- self, self.interfaces))
+ def management_interface(self):
+ if not self._management_interface:
+ raise Exception("No management interface has been defined")
+ return self._management_interface
def execute(self, command, output = False, as_root = False):
raise NotImplementedError