vppapigen: include comments in json

Type: feature
Signed-off-by: Ole Troan <ot@cisco.com>
Change-Id: Ibd796adea734b64d9209c5e18c5b9800cbaf62c6
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

2 files changed, 24 insertions, 11 deletions

diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 9abc5362d80..ce05cbe9b3f 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -89,7 +89,7 @@ class VPPAPILexer:
         "description": "DESCRIPTION",
     }
 
-    tokens = ["STRING_LITERAL", "ID", "NUM"] + list(reserved.values())
+    tokens = ["STRING_LITERAL", "COMMENT", "ID", "NUM"] + list(reserved.values())
 
     t_ignore_LINE_COMMENT = "//.*"
 
@@ -125,9 +125,10 @@ class VPPAPILexer:
         return t
 
     # C or C++ comment (ignore)
-    def t_comment(self, t):
+    def t_COMMENT(self, t):
         r"(/\*(.|\n)*?\*/)|(//.*)"
         t.lexer.lineno += t.value.count("\n")
+        return t
 
     # Error handling rule
     def t_error(self, t):
@@ -301,7 +302,7 @@ class Union(Processable):
 class Define(Processable):
     type = "Define"
 
-    def __init__(self, name, flags, block):
+    def __init__(self, name, flags, block, comment=None):
         self.name = name
         self.flags = flags
         self.block = block
@@ -311,6 +312,7 @@ class Define(Processable):
         self.autoreply = False
         self.autoendian = 0
         self.options = {}
+        self.comment = comment
         for f in flags:
             if f == "dont_trace":
                 self.dont_trace = True
@@ -560,6 +562,7 @@ class VPPAPIParser:
         self.logger = logger
         self.fields = []
         self.revision = revision
+        self.last_comment = None
 
     def _parse_error(self, msg, coord):
         raise ParseError("%s: %s" % (coord, msg))
@@ -600,6 +603,7 @@ class VPPAPIParser:
         | union
         | service
         | paths
+        | comment
         | counters"""
         p[0] = p[1]
 
@@ -747,7 +751,7 @@ class VPPAPIParser:
     def p_define(self, p):
         """define : DEFINE ID '{' block_statements_opt '}' ';'"""
         self.fields = []
-        p[0] = Define(p[2], [], p[4])
+        p[0] = Define(p[2], [], p[4], self.last_comment)
 
     def p_define_flist(self, p):
         """define : flist DEFINE ID '{' block_statements_opt '}' ';'"""
@@ -758,7 +762,7 @@ class VPPAPIParser:
                 self._token_coord(p, 1),
             )
         else:
-            p[0] = Define(p[3], p[1], p[5])
+            p[0] = Define(p[3], p[1], p[5], self.last_comment)
 
     def p_flist(self, p):
         """flist : flag
@@ -865,6 +869,11 @@ class VPPAPIParser:
         """
         p[0] = p[1]
 
+    def p_comment(self, p):
+        """comment : COMMENT"""
+        self.last_comment = p[1]
+        p[0] = []
+
     def p_declaration(self, p):
         """declaration : type_specifier variable_name ';'
         | type_specifier variable_name '[' field_options ']' ';'
@@ -955,26 +964,29 @@ class VPPAPIParser:
     # Error rule for syntax errors
     def p_error(self, p):
         if p:
+            if p.type == "COMMENT":
+                self.parser.errok()
+                return
             self._parse_error("before: %s" % p.value, self._coord(lineno=p.lineno))
         else:
             self._parse_error("At end of input", self.filename)
 
+    def build(self, **kwargs):
+        self.parser = yacc.yacc(module=self, **kwargs)
+
 
 class VPPAPI:
     def __init__(self, debug=False, filename="", logger=None, revision=None):
         self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
-        self.parser = yacc.yacc(
-            module=VPPAPIParser(filename, logger, revision=revision),
-            write_tables=False,
-            debug=debug,
-        )
+        self.parser = VPPAPIParser(filename, logger, revision=revision)
+        self.parser.build(write_tables=False, debug=debug)
         self.logger = logger
         self.revision = revision
         self.filename = filename
 
     def parse_string(self, code, debug=0, lineno=1):
         self.lexer.lineno = lineno
-        return self.parser.parse(code, lexer=self.lexer, debug=debug)
+        return self.parser.parser.parse(code, lexer=self.lexer, debug=debug)
 
     def parse_fd(self, fd, debug=0):
         data = fd.read()
diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py
index 91334468503..ba32e284d81 100644
--- a/src/tools/vppapigen/vppapigen_json.py
+++ b/src/tools/vppapigen/vppapigen_json.py
@@ -77,6 +77,7 @@ def walk_defs(s, is_message=False):
             c = {}
             c["crc"] = "{0:#0{1}x}".format(t.crc, 10)
             c["options"] = t.options
+            c["comment"] = t.comment
             d.append(c)
 
         r.append(d)