Updated LXD image in tutorials + new tutorial + Changed logs for LXD NotFound to warnings + Number of workers is now a function of the number of CPUs

Change-Id: I926ac0e86cd2c594c3f9fce7e20e80a385b4f56a
Signed-off-by: Marcel Enguehard <mengueha+fdio@cisco.com>

6 files changed, 243 insertions, 38 deletions

diff --git a/examples/tutorial/tutorial01.json b/examples/tutorial/tutorial01.json
index 9d1bfac7..d7c716c2 100644
--- a/examples/tutorial/tutorial01.json
+++ b/examples/tutorial/tutorial01.json
@@ -13,36 +13,36 @@
         },
         {
             "type": "LxcImage",
-            "name": "ubuntu1604-cicnsuite-rc1",
+            "name": "ubuntu1604-cicnsuite-rc2",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "prod1",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "prod2",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "core2",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "core1",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "cons1",
             "node": "server"
         },
@@ -50,7 +50,7 @@
             "type": "LxcContainer",
             "name": "cons2",
             "node": "server",
-            "image": "ubuntu1604-cicnsuite-rc1"
+            "image": "ubuntu1604-cicnsuite-rc2"
         },
         {
             "type": "MetisForwarder",
diff --git a/examples/tutorial/tutorial02-dumbell.json b/examples/tutorial/tutorial02-dumbell.json
index 17a4e710..d2274be2 100644
--- a/examples/tutorial/tutorial02-dumbell.json
+++ b/examples/tutorial/tutorial02-dumbell.json
@@ -15,7 +15,7 @@
     },
     {
       "type": "LxcImage",
-      "name": "ubuntu1604-cicnsuite-rc1",	
+      "name": "ubuntu1604-cicnsuite-rc2",
       "node": "server",
       "managed": false
     },
@@ -23,7 +23,7 @@
       "type": "LxcContainer",
       "node": "server",
       "name": "bridge1",
-      "image": "ubuntu1604-cicnsuite-rc1"
+      "image": "ubuntu1604-cicnsuite-rc2"
     },
     {
       "type": "VPP",
@@ -43,7 +43,7 @@
       "type": "LxcContainer",
       "node": "server",
       "name": "core1",
-      "image": "ubuntu1604-cicnsuite-rc1"
+      "image": "ubuntu1604-cicnsuite-rc2"
     },
     {
       "type": "VPP",
@@ -93,7 +93,7 @@
       "type": "LxcContainer",
       "node": "server",
       "name": "core2",
-      "image": "ubuntu1604-cicnsuite-rc1"
+      "image": "ubuntu1604-cicnsuite-rc2"
     },
     {
       "type": "VPP",
@@ -143,7 +143,7 @@
       "type": "LxcContainer",
       "node": "server",
       "name": "bridge2",
-      "image": "ubuntu1604-cicnsuite-rc1"
+      "image": "ubuntu1604-cicnsuite-rc2"
     },
     {
       "type": "VPP",
@@ -158,11 +158,11 @@
       "ip_address" : "172.17.3.23",
       "mac_address": "08:00:27:f2:a8:d9",
       "name": "bridge2-dpdk1"
-    },      
+    },
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "cons1"
     },
     {
@@ -174,7 +174,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "cons2"
     },
     {
@@ -186,7 +186,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "cons3"
     },
     {
@@ -198,7 +198,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "cons4"
     },
     {
@@ -210,7 +210,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "cons5"
     },
     {
@@ -222,7 +222,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "prod1"
     },
     {
@@ -234,7 +234,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "prod2"
     },
     {
@@ -246,7 +246,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "prod3"
     },
     {
@@ -258,7 +258,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "prod4"
     },
     {
@@ -270,7 +270,7 @@
     {
       "type": "LxcContainer",
       "node": "server",
-      "image": "ubuntu1604-cicnsuite-rc1",
+      "image": "ubuntu1604-cicnsuite-rc2",
       "name": "prod5"
     },
     {
diff --git a/examples/tutorial/tutorial03-hetnet.json b/examples/tutorial/tutorial03-hetnet.json
index 42d4292d..4ee25216 100644
--- a/examples/tutorial/tutorial03-hetnet.json
+++ b/examples/tutorial/tutorial03-hetnet.json
@@ -13,12 +13,12 @@
         },
         {
             "type": "LxcImage",
-            "name": "ubuntu1604-cicnsuite-rc1",
+            "name": "ubuntu1604-cicnsuite-rc2",
             "node": "server"
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "cons",
             "node": "server",
             "category": "tablet",
@@ -27,7 +27,7 @@
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "wifi",
             "node": "server",
             "category": "wifi",
@@ -36,7 +36,7 @@
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "lte",
             "node": "server",
             "category": "lte",
@@ -45,7 +45,7 @@
         },
         {
             "type": "LxcContainer",
-            "image": "ubuntu1604-cicnsuite-rc1",
+            "image": "ubuntu1604-cicnsuite-rc2",
             "name": "prod",
             "node": "server",
             "category": "video-server",
diff --git a/examples/tutorial/tutorial04-caching.json b/examples/tutorial/tutorial04-caching.json
new file mode 100644
index 00000000..049f0a43
--- /dev/null
+++ b/examples/tutorial/tutorial04-caching.json
@@ -0,0 +1,203 @@
+{
+    "settings": {
+        "network": "192.168.128.0/24",
+        "ulimit-n": 10000
+    },
+    "resources": [
+        {
+            "type": "Physical",
+            "name": "server",
+            "hostname": "localhost"
+        },
+        {
+            "type": "NetDevice",
+            "device_name": "eth0",
+            "managed": false,
+            "node": "server"
+        },
+        {
+            "type": "LxcImage",
+            "name": "ubuntu1604-cicnsuite-rc2",
+            "node": "server"
+        },
+        {
+            "name": "u1core",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "asr",
+            "type": "LxcContainer",
+            "y": 10,
+            "x": 5,
+            "node": "server"
+        },
+        {
+            "name": "u1srv1",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "meizu",
+            "type": "LxcContainer",
+            "y": 8,
+            "x": 2,
+            "node": "server"
+        },
+        {
+            "name": "u1srv2",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "meizu",
+            "type": "LxcContainer",
+            "y": 12,
+            "x": 2,
+            "node": "server"
+        },
+        {
+            "name": "u2core",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "asr",
+            "type": "LxcContainer",
+            "y": 6,
+            "x": 9,
+            "node": "server"
+        },
+        {
+            "name": "u2srv1",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "meizu",
+            "type": "LxcContainer",
+            "y": 2,
+            "x": 9,
+            "node": "server"
+        },
+        {
+            "name": "u3core",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "asr",
+            "type": "LxcContainer",
+            "y": 12,
+            "x": 13,
+            "node": "server"
+        },
+        {
+            "name": "u3srv1",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "meizu",
+            "type": "LxcContainer",
+            "y": 10,
+            "x": 16,
+            "node": "server"
+        },
+        {
+            "name": "u3srv2",
+            "image": "ubuntu1604-cicnsuite-rc2",
+            "category": "meizu",
+            "type": "LxcContainer",
+            "y": 14,
+            "x": 16,
+            "node": "server"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 2000000,
+            "node": "u1core"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 0,
+            "node": "u1srv1"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 0,
+            "node": "u1srv2"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 2000000,
+            "node": "u2core"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 0,
+            "node": "u2srv1"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 2000000,
+            "node": "u3core"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 0,
+            "node": "u3srv1"
+        },
+        {
+            "type": "WebServer",
+            "prefixes": [
+                "/webserver1"
+            ],
+            "node": "u3srv1"
+        },
+        {
+            "type": "MetisForwarder",
+            "cache_size": 0,
+            "node": "u3srv2"
+        },
+        {
+            "type": "WebServer",
+            "prefixes": [
+                "/webserver2"
+            ],
+            "node": "u3srv2"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u1srv1",
+            "capacity": 100,
+            "src_node": "u1core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u1srv2",
+            "capacity": 100,
+            "src_node": "u1core"
+        },
+       {
+            "type": "Link",
+            "dst_node": "u2srv1",
+            "capacity": 100,
+            "src_node": "u2core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u3srv1",
+            "capacity": 100,
+            "src_node": "u3core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u3srv2",
+            "capacity": 100,
+            "src_node": "u3core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u2core",
+            "src_node": "u1core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u3core",
+            "src_node": "u1core"
+        },
+        {
+            "type": "Link",
+            "dst_node": "u2core",
+            "src_node": "u3core"
+        },
+        {
+            "type": "CentralIP",
+            "ip_routing_strategy": "spt"
+        },
+        {
+            "type": "CentralICN",
+            "face_protocol": "udp4"
+        }
+    ]
+}
diff --git a/vicn/core/resource_mgr.py b/vicn/core/resource_mgr.py
index c71e0444..b750f555 100644
--- a/vicn/core/resource_mgr.py
+++ b/vicn/core/resource_mgr.py
@@ -802,7 +802,7 @@ class ResourceManager(metaclass=Singleton):
                 if ENABLE_LXD_WORKAROUND and \
                         (isinstance(e, LxdNotFound) or isinstance(e, LXDAPIException)):
                     new_state = AttributeState.RESET
-                    log.error('LXD Fix (not found). Reset attribute')
+                    log.warning('LXD Fix (not found). Reset attribute')
                     resource._state.attr_change_success[attribute.name] = True
                 else:
                     log.error('Attribute error {} for resource {}'.format(
@@ -894,7 +894,7 @@ class ResourceManager(metaclass=Singleton):
                     if ENABLE_LXD_WORKAROUND and \
                             (isinstance(attrs, LxdNotFound) or isinstance(attrs, LXDAPIException)):
                         new_state = AttributeState.RESET
-                        log.error('LXD Fix (not found). Reset attribute')
+                        log.warning('LXD Fix (not found). Reset attribute')
                         resource._state.attr_change_success[attribute.name] = True
                     else:
                         self.attr_log(resource, attribute,
@@ -927,7 +927,7 @@ class ResourceManager(metaclass=Singleton):
                     if ENABLE_LXD_WORKAROUND and \
                             (isinstance(attrs, LxdNotFound) or isinstance(attrs, LXDAPIException)):
                         new_state = AttributeState.RESET
-                        log.error('LXD Fix (not found). Reset attribute')
+                        log.warning('LXD Fix (not found). Reset attribute')
                         resource._state.attr_change_success[attribute.name] = True
                     else:
                         log.error('Attribute error {} for resource {}'.format(
@@ -1373,13 +1373,13 @@ class ResourceManager(metaclass=Singleton):
                         # "not found" is the normal exception when the container
                         # does not exists. anyways the bug should only occur
                         # with container.execute(), not container.get()
-                        log.error('LXD Fix (not found). Reset resource')
+                        log.warning('LXD Fix (not found). Reset resource')
                         new_state = ResourceState.INITIALIZED
                     elif ENABLE_LXD_WORKAROUND and isinstance(e, LXDAPIException):
                         # "not found" is the normal exception when the container
                         # does not exists. anyways the bug should only occur
                         # with container.execute(), not container.get()
-                        log.error('LXD Fix (API error). Reset resource')
+                        log.warning('LXD Fix (API error). Reset resource')
                         new_state = ResourceState.INITIALIZED
                     elif isinstance(e, ResourceNotFound):
                         # The resource does not exist
@@ -1403,7 +1403,7 @@ class ResourceManager(metaclass=Singleton):
                     self.log(resource, 'KEYS failed: {}'.format(e))
 
                     if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound):
-                        log.error('LXD Fix (not found). Reset resource')
+                        log.warning('LXD Fix (not found). Reset resource')
                         new_state = ResourceState.CREATED
                         resource._state.change_success = True
                     else:
@@ -1421,12 +1421,12 @@ class ResourceManager(metaclass=Singleton):
                     e = resource._state.change_value
 
                     if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound):
-                        log.error('LXD Fix (not found). Reset resource')
+                        log.warning('LXD Fix (not found). Reset resource')
                         new_state = ResourceState.INITIALIZED
                         resource._state.change_success = True
                     elif ENABLE_LXD_WORKAROUND and \
                             isinstance(e, LXDAPIException):
-                        log.error('LXD Fix (API error). Reset resource')
+                        log.warning('LXD Fix (API error). Reset resource')
                         new_state = ResourceState.INITIALIZED
                         resource._state.change_success = True
                     elif 'File exists' in str(e):
@@ -1453,7 +1453,7 @@ class ResourceManager(metaclass=Singleton):
                     self.log(resource, 'UPDATE failed: {}'.format(e))
 
                     if ENABLE_LXD_WORKAROUND and isinstance(e, LxdNotFound):
-                        log.error('LXD Fix (not found). Reset resource')
+                        log.warning('LXD Fix (not found). Reset resource')
                         new_state = ResourceState.CREATED
                         resource._state.change_success = True
                         resource._state.write_lock.release()
diff --git a/vicn/core/task.py b/vicn/core/task.py
index 5aecb40b..9c92ef4d 100644
--- a/vicn/core/task.py
+++ b/vicn/core/task.py
@@ -22,6 +22,7 @@ import functools
 import logging
 import shlex
 import subprocess
+import os
 
 from vicn.core.scheduling_algebra   import SchedulingAlgebra
 from vicn.core.commands             import ReturnValue
@@ -34,7 +35,8 @@ log = logging.getLogger(__name__)
 EXECUTOR=concurrent.futures.ThreadPoolExecutor
 #EXECUTOR=concurrent.futures.ProcessPoolExecutor
 
-MAX_WORKERS = 50 # None
+#Sets the number of task workers to the number of CPU threads+1
+MAX_WORKERS = os.cpu_count()+1
 
 class BaseTask:
     """Base class for all tasks