telemetry: error message handling

Change-Id: I3bc07072337d45ff24acfbb75cced3b0b052b6f5
Signed-off-by: Viliam Luc <vluc@cisco.com>

8 files changed, 125 insertions, 35 deletions

diff --git a/resources/templates/telemetry/bpf_runtime.yaml b/resources/templates/telemetry/bpf_runtime.yaml
index 88ad7eb64f..bb9d1c70ae 100644
--- a/resources/templates/telemetry/bpf_runtime.yaml
+++ b/resources/templates/telemetry/bpf_runtime.yaml
@@ -2,16 +2,23 @@
 logging:
   version: 1
   formatters:
-    console:
+    console_stdout:
       format: '%(asctime)s - %(name)s - %(message)s'
+    console_stderr:
+      format: '%(message)s'
     prom:
       format: '%(message)s'
   handlers:
-    console:
+    console_stdout:
       class: logging.StreamHandler
       level: INFO
-      formatter: console
+      formatter: console_stdout
       stream: ext://sys.stdout
+    console_stderr:
+      class: logging.StreamHandler
+      level: ERROR
+      formatter: console_stderr
+      stream: ext://sys.stderr
     prom:
       class: logging.handlers.RotatingFileHandler
       level: INFO
@@ -25,7 +32,7 @@ logging:
       propagate: False
   root:
     level: INFO
-    handlers: [console]
+    handlers: [console_stdout, console_stderr]
 scheduler:
   duration: 1
 programs:
diff --git a/resources/templates/telemetry/vpp_clear_stats.yaml b/resources/templates/telemetry/vpp_clear_stats.yaml
index aad916c805..2188588827 100644
--- a/resources/templates/telemetry/vpp_clear_stats.yaml
+++ b/resources/templates/telemetry/vpp_clear_stats.yaml
@@ -2,16 +2,23 @@
 logging:
   version: 1
   formatters:
-    console:
+    console_stdout:
       format: '%(asctime)s - %(name)s - %(message)s'
+    console_stderr:
+      format: '%(message)s'
     prom:
       format: '%(message)s'
   handlers:
-    console:
+    console_stdout:
       class: logging.StreamHandler
       level: INFO
-      formatter: console
+      formatter: console_stdout
       stream: ext://sys.stdout
+    console_stderr:
+      class: logging.StreamHandler
+      level: ERROR
+      formatter: console_stderr
+      stream: ext://sys.stderr
     prom:
       class: logging.handlers.RotatingFileHandler
       level: INFO
@@ -25,7 +32,7 @@ logging:
       propagate: False
   root:
     level: INFO
-    handlers: [console]
+    handlers: [console_stdout, console_stderr]
 scheduler:
   duration: 1
 programs:
diff --git a/resources/templates/telemetry/vpp_runtime.yaml b/resources/templates/telemetry/vpp_runtime.yaml
index a0309a3fb5..b9a2368981 100644
--- a/resources/templates/telemetry/vpp_runtime.yaml
+++ b/resources/templates/telemetry/vpp_runtime.yaml
@@ -2,16 +2,23 @@
 logging:
   version: 1
   formatters:
-    console:
+    console_stdout:
       format: '%(asctime)s - %(name)s - %(message)s'
+    console_stderr:
+      format: '%(message)s'
     prom:
       format: '%(message)s'
   handlers:
-    console:
+    console_stdout:
       class: logging.StreamHandler
       level: INFO
-      formatter: console
+      formatter: console_stdout
       stream: ext://sys.stdout
+    console_stderr:
+      class: logging.StreamHandler
+      level: ERROR
+      formatter: console_stderr
+      stream: ext://sys.stderr
     prom:
       class: logging.handlers.RotatingFileHandler
       level: INFO
@@ -25,7 +32,7 @@ logging:
       propagate: False
   root:
     level: INFO
-    handlers: [console]
+    handlers: [console_stdout, console_stderr]
 scheduler:
   duration: 1
 programs:
diff --git a/resources/templates/telemetry/vpp_show_stats.yaml b/resources/templates/telemetry/vpp_show_stats.yaml
index 06fc229557..dc900e6386 100644
--- a/resources/templates/telemetry/vpp_show_stats.yaml
+++ b/resources/templates/telemetry/vpp_show_stats.yaml
@@ -2,16 +2,23 @@
 logging:
   version: 1
   formatters:
-    console:
+    console_stdout:
       format: '%(asctime)s - %(name)s - %(message)s'
+    console_stderr:
+      format: '%(message)s'
     prom:
       format: '%(message)s'
   handlers:
-    console:
+    console_stdout:
       class: logging.StreamHandler
       level: INFO
-      formatter: console
+      formatter: console_stdout
       stream: ext://sys.stdout
+    console_stderr:
+      class: logging.StreamHandler
+      level: ERROR
+      formatter: console_stderr
+      stream: ext://sys.stderr
     prom:
       class: logging.handlers.RotatingFileHandler
       level: INFO
@@ -25,7 +32,7 @@ logging:
       propagate: False
   root:
     level: INFO
-    handlers: [console]
+    handlers: [console_stdout, console_stderr]
 scheduler:
   duration: 1
 programs:
diff --git a/resources/templates/telemetry/vpp_test_teardown.yaml b/resources/templates/telemetry/vpp_test_teardown.yaml
index 86567afb75..37618bb852 100644
--- a/resources/templates/telemetry/vpp_test_teardown.yaml
+++ b/resources/templates/telemetry/vpp_test_teardown.yaml
@@ -2,16 +2,23 @@
 logging:
   version: 1
   formatters:
-    console:
+    console_stdout:
       format: '%(asctime)s - %(name)s - %(message)s'
+    console_stderr:
+      format: '%(message)s'
     prom:
       format: '%(message)s'
   handlers:
-    console:
+    console_stdout:
       class: logging.StreamHandler
       level: INFO
-      formatter: console
+      formatter: console_stdout
       stream: ext://sys.stdout
+    console_stderr:
+      class: logging.StreamHandler
+      level: ERROR
+      formatter: console_stderr
+      stream: ext://sys.stderr
     prom:
       class: logging.handlers.RotatingFileHandler
       level: INFO
@@ -25,7 +32,7 @@ logging:
       propagate: False
   root:
     level: INFO
-    handlers: [console]
+    handlers: [console_stdout, console_stderr]
 scheduler:
   duration: 0
 programs:
diff --git a/resources/tools/telemetry/bundle_vpp.py b/resources/tools/telemetry/bundle_vpp.py
index 64c4c5e4f8..315360f63b 100644
--- a/resources/tools/telemetry/bundle_vpp.py
+++ b/resources/tools/telemetry/bundle_vpp.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -20,7 +20,7 @@ import struct
 import sys
 
 from vpp_papi.vpp_papi import VPPApiClient as vpp_class
-
+from .constants import Constants
 
 M_RUN_THREAD = (
     r"Thread\s"
@@ -209,8 +209,8 @@ class BundleVpp:
         try:
             self.obj.connect(name=u"telemetry")
         except (ConnectionRefusedError, OSError):
-            getLogger(__name__).error(u"Cannot connect to VPP!")
-            sys.exit(1)
+            getLogger("console_stderr").error(u"Could not connect to VPP!")
+            sys.exit(Constants.err_vpp_connect)
 
         for command in self.code.splitlines():
             api_name = u"cli_inband"
@@ -223,7 +223,11 @@ class BundleVpp:
         """
         Detach from VPP.
         """
-        self.obj.disconnect()
+        try:
+            self.obj.disconnect()
+        except (ConnectionRefusedError, OSError):
+            getLogger("console_stderr").error(u"Could not disconnect from VPP!")
+            sys.exit(Constants.err_vpp_disconnect)
 
     def fetch_data(self):
         """
@@ -234,8 +238,12 @@ class BundleVpp:
                 papi_fn = getattr(self.obj.api, command[u"api_name"])
                 getLogger(__name__).info(command[u"api_args"][u"cmd"])
                 replies = papi_fn(**command[u"api_args"])
-            except (AttributeError, IOError, struct.error) as err:
-                raise AssertionError(err)
+            except (AssertionError, AttributeError, IOError, struct.error):
+                getLogger("console_stderr").error(
+                    f"Failed when executing command: "
+                    f"{command['api_args']['cmd']}"
+                )
+                sys.exit(Constants.err_vpp_execute)
 
             if not isinstance(replies, list):
                 replies = [replies]
@@ -253,12 +261,22 @@ class BundleVpp:
         Post process command reply.
         """
         for command in zip(self.api_command_list, self.api_replies_list):
-            self_fn = command[0][u"api_args"][u"cmd"]
+            self_fn = command[0][u"api_args"][u"cmd"].replace(u" ", u"_")
+            self_method_list = [meth for meth in dir(self)
+                           if callable(getattr(self, meth)) and
+                           meth.startswith('__') is False]
+            if self_fn not in self_method_list:
+                continue
             try:
-                self_fn = getattr(self, self_fn.replace(u" ", u"_"))
+                self_fn = getattr(self, self_fn)
                 self_fn(command[1].reply)
             except AttributeError:
                 pass
+            except (KeyError, ValueError, TypeError) as e:
+                getLogger("console_stderr").error(
+                    f"Failed when processing data. Error message {e}"
+                )
+                sys.exit(Constants.err_telemetry_process)
 
     def show_interface(self, reply):
         """
@@ -441,7 +459,7 @@ class BundleVpp:
             item[u"name"] = metric
             labels[u"version"] = version
             item[u"labels"] = labels
-            item[u"value"] = 1.0
+            item[u"value"] = {}
             self.serializer.serialize(
                 metric=metric, labels=labels, item=item
             )
diff --git a/resources/tools/telemetry/constants.py b/resources/tools/telemetry/constants.py
new file mode 100644
index 0000000000..fda558b0b4
--- /dev/null
+++ b/resources/tools/telemetry/constants.py
@@ -0,0 +1,37 @@
+# Copyright (c) 2022 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Constants used in telemetry.
+
+"Constant" means a value that keeps its value since initialization. The value
+does not need to be hard coded here, but can be read from environment variables.
+"""
+
+class Constants:
+    """Constants used in telemetry.
+    1-10: Telemetry errors
+    11-50: VPP bundle error
+    51-100: Linux bundle errors"""
+
+    # Failed when processing data
+    err_telemetry_process = 1
+
+    # Could not connect to VPP
+    err_vpp_connect = 11
+
+    # Could not disconnect from VPP
+    err_vpp_disconnect = 12
+
+    # Failed when executing command
+    err_vpp_execute = 13
+
diff --git a/resources/tools/telemetry/metrics.py b/resources/tools/telemetry/metrics.py
index 281760183b..7a22acfd1b 100644
--- a/resources/tools/telemetry/metrics.py
+++ b/resources/tools/telemetry/metrics.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 Cisco and/or its affiliates.
+# Copyright (c) 2022 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -340,7 +340,7 @@ class MetricBase:
 
     def samples(self):
         """
-        Returns samples wheter an object is parent or child.
+        Returns samples whether an object is parent or child.
 
         :returns: List of Metric objects with values.
         :rtype: list
@@ -602,9 +602,9 @@ class Info(MetricBase):
         """
         Set info to the given value.
 
-        :param amount: Value to set.
-        :type amount: int or float
-        :raises ValueError: If lables are overlapping.
+        :param value: Value to set.
+        :type value: int or float
+        :raises ValueError: If labels are overlapping.
         """
         if self._labelname_set.intersection(value.keys()):
             raise ValueError(