From 8b0ac576c778b380ff341ce4022b4d1f83aa7021 Mon Sep 17 00:00:00 2001 From: Marcel Enguehard Date: Wed, 12 Apr 2017 10:44:55 +0200 Subject: Updated LXD image in tutorials + new tutorial + Changed logs for LXD NotFound to warnings + Number of workers is now a function of the number of CPUs Change-Id: I926ac0e86cd2c594c3f9fce7e20e80a385b4f56a Signed-off-by: Marcel Enguehard --- examples/tutorial/tutorial01.json | 14 +-- examples/tutorial/tutorial02-dumbell.json | 32 ++--- examples/tutorial/tutorial03-hetnet.json | 10 +- examples/tutorial/tutorial04-caching.json | 203 ++++++++++++++++++++++++++++++ vicn/core/resource_mgr.py | 18 +-- vicn/core/task.py | 4 +- 6 files changed, 243 insertions(+), 38 deletions(-) create mode 100644 examples/tutorial/tutorial04-caching.json diff --git a/examples/tutorial/tutorial01.json b/examples/tutorial/tutorial01.json index 9d1bfac7..d7c716c2 100644 --- a/examples/tutorial/tutorial01.json +++ b/examples/tutorial/tutorial01.json @@ -13,36 +13,36 @@ }, { "type": "LxcImage", - "name": "ubuntu1604-cicnsuite-rc1", + "name": "ubuntu1604-cicnsuite-rc2", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod1", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod2", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "core2", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "core1", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons1", "node": "server" }, @@ -50,7 +50,7 @@ "type": "LxcContainer", "name": "cons2", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1" + "image": "ubuntu1604-cicnsuite-rc2" }, { "type": "MetisForwarder", diff --git a/examples/tutorial/tutorial02-dumbell.json b/examples/tutorial/tutorial02-dumbell.json index 17a4e710..d2274be2 100644 --- a/examples/tutorial/tutorial02-dumbell.json +++ b/examples/tutorial/tutorial02-dumbell.json @@ -15,7 +15,7 @@ }, { "type": "LxcImage", - "name": "ubuntu1604-cicnsuite-rc1", + "name": "ubuntu1604-cicnsuite-rc2", "node": "server", "managed": false }, @@ -23,7 +23,7 @@ "type": "LxcContainer", "node": "server", "name": "bridge1", - "image": "ubuntu1604-cicnsuite-rc1" + "image": "ubuntu1604-cicnsuite-rc2" }, { "type": "VPP", @@ -43,7 +43,7 @@ "type": "LxcContainer", "node": "server", "name": "core1", - "image": "ubuntu1604-cicnsuite-rc1" + "image": "ubuntu1604-cicnsuite-rc2" }, { "type": "VPP", @@ -93,7 +93,7 @@ "type": "LxcContainer", "node": "server", "name": "core2", - "image": "ubuntu1604-cicnsuite-rc1" + "image": "ubuntu1604-cicnsuite-rc2" }, { "type": "VPP", @@ -143,7 +143,7 @@ "type": "LxcContainer", "node": "server", "name": "bridge2", - "image": "ubuntu1604-cicnsuite-rc1" + "image": "ubuntu1604-cicnsuite-rc2" }, { "type": "VPP", @@ -158,11 +158,11 @@ "ip_address" : "172.17.3.23", "mac_address": "08:00:27:f2:a8:d9", "name": "bridge2-dpdk1" - }, + }, { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons1" }, { @@ -174,7 +174,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons2" }, { @@ -186,7 +186,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons3" }, { @@ -198,7 +198,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons4" }, { @@ -210,7 +210,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons5" }, { @@ -222,7 +222,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod1" }, { @@ -234,7 +234,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod2" }, { @@ -246,7 +246,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod3" }, { @@ -258,7 +258,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod4" }, { @@ -270,7 +270,7 @@ { "type": "LxcContainer", "node": "server", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod5" }, { diff --git a/examples/tutorial/tutorial03-hetnet.json b/examples/tutorial/tutorial03-hetnet.json index 42d4292d..4ee25216 100644 --- a/examples/tutorial/tutorial03-hetnet.json +++ b/examples/tutorial/tutorial03-hetnet.json @@ -13,12 +13,12 @@ }, { "type": "LxcImage", - "name": "ubuntu1604-cicnsuite-rc1", + "name": "ubuntu1604-cicnsuite-rc2", "node": "server" }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "cons", "node": "server", "category": "tablet", @@ -27,7 +27,7 @@ }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "wifi", "node": "server", "category": "wifi", @@ -36,7 +36,7 @@ }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "lte", "node": "server", "category": "lte", @@ -45,7 +45,7 @@ }, { "type": "LxcContainer", - "image": "ubuntu1604-cicnsuite-rc1", + "image": "ubuntu1604-cicnsuite-rc2", "name": "prod", "node": "server", "category": "video-server", diff --git a/examples/tutorial/tutorial04-caching.json b/examples/tutorial/tutorial04-caching.json new file mode 100644 index 00000000..049f0a43 --- /dev/null +++ b/examples/tutorial/tutorial04-caching.json @@ -0,0 +1,203 @@ +{ + "settings": { + "network": "192.168.128.0/24", + "ulimit-n": 10000 + }, + "resources": [ + { + "type": "Physical", + "name": "server", + "hostname": "localhost" + }, + { + "type": "NetDevice", + "device_name": "eth0", + "managed": false, + "node": "server" + }, + { + "type": "LxcImage", + "name": "ubuntu1604-cicnsuite-rc2", + "node": "server" + }, + { + "name": "u1core", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "asr", + "type": "LxcContainer", + "y": 10, + "x": 5, + "node": "server" + }, + { + "name": "u1srv1", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "meizu", + "type": "LxcContainer", + "y": 8, + "x": 2, + "node": "server" + }, + { + "name": "u1srv2", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "meizu", + "type": "LxcContainer", + "y": 12, + "x": 2, + "node": "server" + }, + { + "name": "u2core", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "asr", + "type": "LxcContainer", + "y": 6, + "x": 9, + "node": "server" + }, + { + "name": "u2srv1", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "meizu", + "type": "LxcContainer", + "y": 2, + "x": 9, + "node": "server" + }, + { + "name": "u3core", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "asr", + "type": "LxcContainer", + "y": 12, + "x": 13, + "node": "server" + }, + { + "name": "u3srv1", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "meizu", + "type": "LxcContainer", + "y": 10, + "x": 16, + "node": "server" + }, + { + "name": "u3srv2", + "image": "ubuntu1604-cicnsuite-rc2", + "category": "meizu", + "type": "LxcContainer", + "y": 14, + "x": 16, + "node": "server" + }, + { + "type": "MetisForwarder", + "cache_size": 2000000, + "node": "u1core" + }, + { + "type": "MetisForwarder", + "cache_size": 0, + "node": "u1srv1" + }, + { + "type": "MetisForwarder", + "cache_size": 0, + "node": "u1srv2" + }, + { + "type": "MetisForwarder", + "cache_size": 2000000, + "node": "u2core" + }, + { + "type": "MetisForwarder", + "cache_size": 0, + "node": "u2srv1" + }, + { + "type": "MetisForwarder", + "cache_size": 2000000, + "node": "u3core" + }, + { + "type": "MetisForwarder", + "cache_size": 0, + "node": "u3srv1" + }, + { + "type": "WebServer", + "prefixes": [ + "/webserver1" + ], + "node": "u3srv1" + }, + { + "type": "MetisForwarder", + "cache_size": 0, + "node": "u3srv2" + }, + { + "type": "WebServer", + "prefixes": [ + "/webserver2" + ], + "node": "u3srv2" + }, + { + "type": "Link", + "dst_node": "u1srv1", + "capacity": 100, + "src_node": "u1core" + }, + { + "type": "Link", + "dst_node": "u1srv2", + "capacity": 100, + "src_node": "u1core" + }, + { + "type": "Link", + "dst_node": "u2srv1", + "capacity": 100, + "src_node": "u2core" + }, + { + "type": "Link", + "dst_node": "u3srv1", + "capacity": 100, + "src_node": "u3core" + }, + { + "type": "Link", + "dst_node": "u3srv2", + "capacity": 100, + "src_node": "u3core" + }, + { + "type": "Link", + "dst_node": "u2core", + "src_node": "u1core" + }, + { + "type": "Link", + "dst_node": "u3core", + "src_node": "u1core" + }, + { + "type": "Link", + "dst_node": "u2core", + "src_node": "u3core" + }, + { + "type": "CentralIP", + "ip_routing_strategy": "spt" + }, + { + "type": "CentralICN", + "face_protocol": "udp4" + } + ] +} diff --git a/vicn/core/resource_mgr.py b/vicn/core/resource_mgr.py index c71e0444..b750f555 100644 --- a/vicn/core/resource_mgr.py +++ b/vicn/core/resource_mgr.py @@ -802,7 +802,7 @@ class ResourceManager(metaclass=Singleton): if ENABLE_LXD_WORKAROUND and \ (isinstance(e, LxdNotFound) or isinstance(e, LXDAPIException)): new_state = AttributeState.RESET - log.error('LXD Fix (not found). Reset attribute') + log.warning('LXD Fix (not found). Reset attribute') resource._state.attr_change_success[attribute.name] = True else: log.error('Attribute error {} for resource {}'.format( @@ -894,7 +894,7 @@ class ResourceManager(metaclass=Singleton): if ENABLE_LXD_WORKAROUND and \ (isinstance(attrs, LxdNotFound) or isinstance(attrs, LXDAPIException)): new_state = AttributeState.RESET - log.error('LXD Fix (not found). Reset attribute') + log.warning('LXD Fix (not found). Reset attribute') resource._state.attr_change_success[attribute.name] = True else: self.attr_log(resource, attribute, @@ -927,7 +927,7 @@ class ResourceManager(metaclass=Singleton): if ENABLE_LXD_WORKAROUND and \ (isinstance(attrs, LxdNotFound) or isinstance(attrs, LXDAPIException)): new_state = AttributeState.RESET - log.error('LXD Fix (not found). Reset attribute') + log.warning('LXD Fix (not found). Reset attribute') resource._state.attr_change_success[attribute.name] = True else: log.error('Attribute error {} for resource {}'.format( @@ -1373,13 +1373,13 @@ class ResourceManager(metaclass=Singleton): # "not found" is the normal exception when the container # does not exists. anyways the bug should only occur # with container.execute(), not container.get() - log.error('LXD Fix (not found). Reset resource') + log.warning('LXD Fix (not found). Reset resource') new_state = ResourceState.INITIALIZED elif ENABLE_LXD_WORKAROUND and isinstance(e, LXDAPIException): # "not found" is the normal exception when the container # does not exists. anyways the bug should only occur # with container.execute(), not container.get() - log.error('LXD Fix (API error). Reset resource') + log.warning('LXD Fix (API error). Reset resource') new_state = ResourceState.INITIALIZED elif isinstance(e, ResourceNotFound): # The resource does not exist @@ -1403,7 +1403,7 @@ class ResourceManager(metaclass=Singleton): self.log(resource, 'KEYS failed: {}'.format(e)) if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound): - log.error('LXD Fix (not found). Reset resource') + log.warning('LXD Fix (not found). Reset resource') new_state = ResourceState.CREATED resource._state.change_success = True else: @@ -1421,12 +1421,12 @@ class ResourceManager(metaclass=Singleton): e = resource._state.change_value if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound): - log.error('LXD Fix (not found). Reset resource') + log.warning('LXD Fix (not found). Reset resource') new_state = ResourceState.INITIALIZED resource._state.change_success = True elif ENABLE_LXD_WORKAROUND and \ isinstance(e, LXDAPIException): - log.error('LXD Fix (API error). Reset resource') + log.warning('LXD Fix (API error). Reset resource') new_state = ResourceState.INITIALIZED resource._state.change_success = True elif 'File exists' in str(e): @@ -1453,7 +1453,7 @@ class ResourceManager(metaclass=Singleton): self.log(resource, 'UPDATE failed: {}'.format(e)) if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound): - log.error('LXD Fix (not found). Reset resource') + log.warning('LXD Fix (not found). Reset resource') new_state = ResourceState.CREATED resource._state.change_success = True resource._state.write_lock.release() diff --git a/vicn/core/task.py b/vicn/core/task.py index 5aecb40b..9c92ef4d 100644 --- a/vicn/core/task.py +++ b/vicn/core/task.py @@ -22,6 +22,7 @@ import functools import logging import shlex import subprocess +import os from vicn.core.scheduling_algebra import SchedulingAlgebra from vicn.core.commands import ReturnValue @@ -34,7 +35,8 @@ log = logging.getLogger(__name__) EXECUTOR=concurrent.futures.ThreadPoolExecutor #EXECUTOR=concurrent.futures.ProcessPoolExecutor -MAX_WORKERS = 50 # None +#Sets the number of task workers to the number of CPU threads+1 +MAX_WORKERS = os.cpu_count()+1 class BaseTask: """Base class for all tasks -- cgit 1.2.3-korg