JSON: Implement latest changes

Change-Id: Ife5feacad6885427066a5b28978f25965394b557
Signed-off-by: Tibor Frank <tifrank@cisco.com>

1 files changed, 82 insertions, 153 deletions

diff --git a/resources/tools/presentation/convert_xml_json.py b/resources/tools/presentation/convert_xml_json.py
index e9ccca0b63..61c6e84a98 100644
--- a/resources/tools/presentation/convert_xml_json.py
+++ b/resources/tools/presentation/convert_xml_json.py
@@ -14,7 +14,7 @@
 """Convert output_info.xml files into JSON structures.
 
 Version: 0.1.0
-Date:    8th June 2021
+Date:    22nd June 2021
 
 The json structure is defined in https://gerrit.fd.io/r/c/csit/+/28992
 """
@@ -72,97 +72,37 @@ class JSONData:
         """
         return self._data
 
-    def add_element(self, value, path_to_value):
-        """Add an element to the json structure.
+    def update(self, kwargs):
+        """Update the data with new data from the dictionary.
 
-        :param value: Element value.
-        :param path_to_value: List of tuples where the first item is the element
-            on the path and the second one is its type.
-        :type value: dict, list, str, int, float, bool
-        :type path_to_value: list
-        :raises: IndexError if the path is empty.
-        :raises: TypeError if the val is of not supported type.
+        :param kwargs: Key value pairs to be added to the data.
+        :type kwargs: dict
         """
+        self._data.update(kwargs)
 
-        def _add_element(val, path, structure):
-            """Add an element to the given path.
-
-            :param val: Element value.
-            :param path: List of tuples where the first item is the element
-            on the path and the second one is its type.
-            :param structure: The structure where the element is added.
-            :type val: dict, list, str, int, float, bool
-            :type path: list
-            :type structure: dict
-            :raises TypeError if there is a wrong type in the path.
-            """
-            if len(path) == 1:
-                if isinstance(structure, dict):
-                    if path[0][1] is dict:
-                        if path[0][0] not in structure:
-                            structure[path[0][0]] = dict()
-                        structure[path[0][0]].update(val)
-                    elif path[0][1] is list:
-                        if path[0][0] not in structure:
-                            structure[path[0][0]] = list()
-                        if isinstance(val, list):
-                            structure[path[0][0]].extend(val)
-                        else:
-                            structure[path[0][0]].append(val)
-                    else:
-                        structure[path[0][0]] = val
-                elif isinstance(structure, list):
-                    if path[0][0] == -1 or path[0][0] >= len(structure):
-                        if isinstance(val, list):
-                            structure.extend(val)
-                        else:
-                            structure.append(val)
-                    else:
-                        structure[path[0][0]] = val
-                return
-
-            if isinstance(structure, dict):
-                if path[0][1] is dict:
-                    if path[0][0] not in structure:
-                        structure[path[0][0]] = dict()
-                elif path[0][1] is list:
-                    if path[0][0] not in structure:
-                        structure[path[0][0]] = list()
-            elif isinstance(structure, list):
-                if path[0][0] == -1 or path[0][0] >= len(structure):
-                    if path[0][1] is list:
-                        structure.append(list())
-                    elif path[0][1] is dict:
-                        structure.append(dict())
-                    else:
-                        structure.append(0)
-                    path[0][0] = len(structure) - 1
-            else:
-                raise TypeError(
-                    u"Only the last item in the path can be different type "
-                    u"then list or dictionary."
-                )
-            _add_element(val, path[1:], structure[path[0][0]])
-
-        if not isinstance(value, (dict, list, str, int, float, bool)):
-            raise TypeError(
-                u"The value must be one of these types: dict, list, str, int, "
-                u"float, bool.\n"
-                f"Value: {value}\n"
-                f"Path: {path_to_value}"
-            )
-        _add_element(deepcopy(value), path_to_value, self._data)
-
-    def get_element(self, path):
-        """Get the element specified by the path.
-
-        :param path: List of keys and indices to the requested element or
-            sub-tree.
-        :type path: list
-        :returns: Element specified by the path.
-        :rtype: any
+    def set_key(self, key, val):
+        """Setter.
+
+        :param key: The key to be updated / added.
+        :param val: The key value.
+        :type key: str
+        :type val: object
+        """
+        self._data[key] = deepcopy(val)
+
+    def add_to_list(self, key, val):
+        """Add an item to the list identified by key.
+
+        :param key: The key identifying the list.
+        :param val: The val to be appended to the list. If val is a list,
+            extend is used.
         """
-        raise NotImplementedError
+        if self._data.get(key, None) is None:
+            self._data[key] = list()
+        if isinstance(val, list):
+            self._data[key].extend(val)
+        else:
+            self._data[key].append(val)
 
     def dump(self, file_out, indent=None):
         """Write JSON data to a file.
@@ -207,53 +147,29 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
     :type metadata: dict
     """
 
-    p_metadata = [(u"metadata", dict), ]
-    p_test = [(u"test", dict), ]
-    p_log = [(u"log", list), (-1, list)]
-
     data = JSONData(template=template)
 
-    data.add_element({u"suite-id": metadata.pop(u"suite-id", u"")}, p_metadata)
-    data.add_element(
-        {u"suite-doc": metadata.pop(u"suite-doc", u"")}, p_metadata
-    )
-    data.add_element({u"testbed": metadata.pop(u"testbed", u"")}, p_metadata)
-    data.add_element(
-        {u"sut-version": metadata.pop(u"sut-version", u"")}, p_metadata
-    )
-
-    data.add_element({u"test-id": tid}, p_test)
+    data.update(metadata)
+    data.set_key(u"test_id", tid)
     t_type = in_data.get(u"type", u"")
     t_type = u"NDRPDR" if t_type == u"CPS" else t_type  # It is NDRPDR
-    data.add_element({u"test-type": t_type}, p_test)
+    data.set_key(u"test_type", t_type)
     tags = in_data.get(u"tags", list())
-    data.add_element({u"tags": tags}, p_test)
-    data.add_element(
-        {u"documentation": in_data.get(u"documentation", u"")}, p_test
-    )
-    data.add_element({u"message": in_data.get(u"msg", u"")}, p_test)
-    execution = {
-        u"start_time": in_data.get(u"starttime", u""),
-        u"end_time": in_data.get(u"endtime", u""),
-        u"status": in_data.get(u"status", u"FAILED"),
-    }
-    execution.update(metadata)
-    data.add_element({u"execution": execution}, p_test)
-
-    log_item = {
-        u"source": {
-            u"type": u"node",
-            u"id": ""
-        },
-        u"msg-type": u"",
-        u"log-level": u"INFO",
-        u"timestamp": in_data.get(u"starttime", u""),  # replacement
-        u"msg": u"",
-        u"data": []
-    }
+    data.set_key(u"tags", tags)
+    data.set_key(u"documentation", in_data.get(u"documentation", u""))
+    data.set_key(u"message", in_data.get(u"msg", u""))
+    data.set_key(u"start_time", in_data.get(u"starttime", u""))
+    data.set_key(u"end_time", in_data.get(u"endtime", u""))
+    data.set_key(u"status", in_data.get(u"status", u"FAILED"))
+    sut_type = u""
+    if u"vpp" in tid:
+        sut_type = u"vpp"
+    elif u"dpdk" in tid:
+        sut_type = u"dpdk"
+    data.set_key(u"sut_type", sut_type)
 
     # Process configuration history:
-    in_papi = deepcopy(in_data.get(u"conf-history", None))
+    in_papi = deepcopy(in_data.get(u"conf_history", None))
     if in_papi:
         regex_dut = re.compile(r'\*\*DUT(\d):\*\*')
         node_id = u"dut1"
@@ -264,20 +180,33 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
             if groups:
                 node_id = f"dut{groups.group(1)}"
             else:
-                log_item[u"source"][u"id"] = node_id
-                log_item[u"msg-type"] = u"papi"
-                log_item[u"msg"] = line
-                data.add_element(log_item, p_log)
+                data.add_to_list(
+                    u"log",
+                    {
+                        u"source_type": u"node",
+                        u"source_id": node_id,
+                        u"msg_type": u"papi",
+                        u"log_level": u"INFO",
+                        u"timestamp": in_data.get(u"starttime", u""),
+                        u"msg": line,
+                        u"data": list()
+                    }
+                )
 
     # Process show runtime:
     in_sh_run = deepcopy(in_data.get(u"show-run", None))
     if in_sh_run:
         # Transform to openMetrics format
         for key, val in in_sh_run.items():
-            log_item[u"source"][u"id"] = key
-            log_item[u"msg-type"] = u"metric"
-            log_item[u"msg"] = u"show-runtime"
-            log_item[u"data"] = list()
+            log_item = {
+                u"source_type": u"node",
+                u"source_id": key,
+                u"msg_type": u"metric",
+                u"log_level": u"INFO",
+                u"timestamp": in_data.get(u"starttime", u""),
+                u"msg": u"show_runtime",
+                u"data": list()
+            }
             for item in val.get(u"runtime", list()):
                 for metric, m_data in item.items():
                     if metric == u"name":
@@ -290,12 +219,12 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
                                 u"labels": {
                                     u"host": val.get(u"host", u""),
                                     u"socket": val.get(u"socket", u""),
-                                    u"graph-node": item.get(u"name", u""),
-                                    u"thread-id": str(idx)
+                                    u"graph_node": item.get(u"name", u""),
+                                    u"thread_id": str(idx)
                                 }
                             }
                         )
-            data.add_element(log_item, p_log)
+            data.add_to_list(u"log", log_item)
 
     # Process results:
     results = dict()
@@ -338,23 +267,23 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
             },
             u"latency": {
                 u"forward": {
-                    u"pdr-90": in_data.get(u"latency", dict()).
+                    u"pdr_90": in_data.get(u"latency", dict()).
                                get(u"PDR90", dict()).get(u"direction1", u"NaN"),
-                    u"pdr-50": in_data.get(u"latency", dict()).
+                    u"pdr_50": in_data.get(u"latency", dict()).
                                get(u"PDR50", dict()).get(u"direction1", u"NaN"),
-                    u"pdr-10": in_data.get(u"latency", dict()).
+                    u"pdr_10": in_data.get(u"latency", dict()).
                                get(u"PDR10", dict()).get(u"direction1", u"NaN"),
-                    u"pdr-0": in_data.get(u"latency", dict()).
+                    u"pdr_0": in_data.get(u"latency", dict()).
                               get(u"LAT0", dict()).get(u"direction1", u"NaN")
                 },
                 u"reverse": {
-                    u"pdr-90": in_data.get(u"latency", dict()).
+                    u"pdr_90": in_data.get(u"latency", dict()).
                                get(u"PDR90", dict()).get(u"direction2", u"NaN"),
-                    u"pdr-50": in_data.get(u"latency", dict()).
+                    u"pdr_50": in_data.get(u"latency", dict()).
                                get(u"PDR50", dict()).get(u"direction2", u"NaN"),
-                    u"pdr-10": in_data.get(u"latency", dict()).
+                    u"pdr_10": in_data.get(u"latency", dict()).
                                get(u"PDR10", dict()).get(u"direction2", u"NaN"),
-                    u"pdr-0": in_data.get(u"latency", dict()).
+                    u"pdr_0": in_data.get(u"latency", dict()).
                               get(u"LAT0", dict()).get(u"direction2", u"NaN")
                 }
             }
@@ -369,7 +298,7 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
         }
     elif t_type == "SOAK":
         results = {
-            u"critical-rate": {
+            u"critical_rate": {
                 u"lower": in_data.get(u"throughput", dict()).
                           get(u"LOWER", u"NaN"),
                 u"upper": in_data.get(u"throughput", dict()).
@@ -387,7 +316,7 @@ def _export_test_from_xml_to_json(tid, in_data, out, template, metadata):
         }
     else:
         pass
-    data.add_element({u"results": results}, p_test)
+    data.set_key(u"results", results)
 
     data.dump(out, indent=u"    ")
 
@@ -456,12 +385,12 @@ def convert_xml_to_json(spec, data):
                     {
                         u"ci": u"jenkins.fd.io",
                         u"job": job,
-                        u"build": build_nr,
-                        u"suite-id": suite_id,
-                        u"suite-doc": build[u"suites"].get(suite_id, dict()).
+                        u"build_number": build_nr,
+                        u"suite_id": suite_id,
+                        u"suite_doc": build[u"suites"].get(suite_id, dict()).
                                       get(u"doc", u""),
                         u"testbed": build[u"metadata"].get(u"testbed", u""),
-                        u"sut-version": build[u"metadata"].get(u"version", u"")
+                        u"sut_version": build[u"metadata"].get(u"version", u"")
                     }
                 )