docs: fix 'make doxygen' under python3

The 'make doxygen' component has this cool vpp specific customization called siphon.
This updates the siphon component so that 'make doxygen' works with python3.

Needed-By:  https://gerrit.fd.io/r/23159

Type: docs

Change-Id: Ie29f1602bf3460b637058acbb0a2f19b128a8824
Signed-off-by: Paul Vinciguerra <pvinci@vinciconsulting.com>

13 files changed, 105 insertions, 74 deletions

diff --git a/doxygen/Makefile b/doxygen/Makefile
index f1bee5b847c..3922ab33e76 100644
--- a/doxygen/Makefile
+++ b/doxygen/Makefile
@@ -33,9 +33,9 @@ endif
 OS_ID ?= $(shell grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 
 # Package dependencies
-DOC_DEB_DEPENDS = doxygen graphviz python-pyparsing python-jinja2
-DOC_RPM_DEPENDS = doxygen graphviz pyparsing python-jinja2
-DOC_SUSE_RPM_DEPENDS = doxygen graphviz python-pyparsing python-Jinja2
+DOC_DEB_DEPENDS = doxygen graphviz python3-pyparsing python3-jinja2
+DOC_RPM_DEPENDS = doxygen graphviz python3-pyparsing python3-jinja2
+DOC_SUSE_RPM_DEPENDS = doxygen graphviz python3-pyparsing python3-Jinja2
 DOC_MAC_BIN_DEPENDS = doxygen dot git
 DOC_MAC_PY_DEPENDS = pyparsing jinja2
 
diff --git a/doxygen/filter_api.py b/doxygen/filter_api.py
index 3e2aaaec290..484881439b8 100755
--- a/doxygen/filter_api.py
+++ b/doxygen/filter_api.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +15,8 @@
 
 # Filter for vpe.api to make it Doxygenish.
 
-import sys, re
+import re
+import sys
 
 if len(sys.argv) < 2:
     sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
@@ -23,15 +24,18 @@ if len(sys.argv) < 2:
 
 patterns = [
     # Search for "define" blocks and treat them as structs
-    ( re.compile(r"^.*(manual_.[^\s]+\s+)?define\s+(?P<name>[^\s]+)"), r"typedef struct vl_api_\g<name>_t"),
+    (re.compile(r"^.*(manual_.[^\s]+\s+)?define\s+(?P<name>[^\s]+)"),
+     r"typedef struct vl_api_\g<name>_t"),
 
     # For every "brief" statement at the start of a comment block, add an
     # xref with whatever is on the same line. This gives us an index page
     # with all the API methods in one place.
     # XXX Commented out for now; works but duplicates the brief text in the
     # struct documentation
-    #( re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)(\*/)$"), r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),  # capture inline comment close
-    #( re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)$"), r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),
+    # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)(\*/)$"),
+    #  r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),  # capture inline comment close
+    # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)$"),
+    #  r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),
 
     # Since structs don't have params, replace @param with @tparam
     ( re.compile("[\\@]param\\b"), "@tparam"),
diff --git a/doxygen/filter_c.py b/doxygen/filter_c.py
index 30b933ba79e..897f9f6d0b3 100755
--- a/doxygen/filter_c.py
+++ b/doxygen/filter_c.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/doxygen/filter_h.py b/doxygen/filter_h.py
index 967388d5743..0891fa708e1 100755
--- a/doxygen/filter_h.py
+++ b/doxygen/filter_h.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +15,9 @@
 
 # Filter for .c files to make various preprocessor tricks Doxygenish
 
-import os, sys, re
+import os
+import re
+import sys
 
 if len(sys.argv) < 2:
     sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
@@ -24,8 +26,9 @@ if len(sys.argv) < 2:
 replace_patterns = [
     # Search for CLIB_PAD_FROM_TO(...); and replace with padding
     # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
-    ( re.compile("(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),\s*(?P<to>[^)]+)[)]"),
-        r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]" ),
+    (re.compile(r"(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),"
+                r"\s*(?P<to>[^)]+)[)]"),
+     r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]"),
 
 ]
 
@@ -42,7 +45,7 @@ with open(filename) as fd:
 
     for line in fd:
         line_num += 1
-        str = line[:-1] # filter \n
+        str = line[:-1]  # filter \n
 
         # Look for search/replace patterns
         for p in replace_patterns:
diff --git a/doxygen/siphon-generate b/doxygen/siphon-generate
index bdfd58d8bbb..9b69c52cf17 100755
--- a/doxygen/siphon-generate
+++ b/doxygen/siphon-generate
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +17,10 @@
 # off into another file for later parsing; ostensibly to generate
 # documentation from struct initializer data.
 
-import os, sys, argparse, logging
+import argparse
+import logging
+import os
+
 import siphon
 
 DEFAULT_LOGFILE = None
@@ -27,26 +30,28 @@ DEFAULT_PREFIX = os.getcwd()
 
 ap = argparse.ArgumentParser()
 ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
-        help="Log file [%s]" % DEFAULT_LOGFILE)
+                help="Log file [%s]" % DEFAULT_LOGFILE)
 ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
-        choices=["debug", "info", "warning", "error", "critical"],
-        help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+                choices=["debug", "info", "warning", "error", "critical"],
+                help="Logging level [%s]" % DEFAULT_LOGLEVEL)
 
 ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
-        help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT)
+                help="Output directory for .siphon files [%s]" %
+                     DEFAULT_OUTPUT)
 ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
-        help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
+                help="Prefix to strip from input pathnames [%s]" %
+                     DEFAULT_PREFIX)
 ap.add_argument("input", nargs='+', metavar="input_file",
-        help="Input C source files")
+                help="Input C source files")
 args = ap.parse_args()
 
 logging.basicConfig(filename=args.log_file,
-        level=getattr(logging, args.log_level.upper(), None))
+                    level=getattr(logging, args.log_level.upper(), None))
 log = logging.getLogger("siphon_generate")
 
 
 generate = siphon.generate.Generate(output_directory=args.output,
-    input_prefix=args.input_prefix)
+                                    input_prefix=args.input_prefix)
 
 # Pre-process file names in case they indicate a file with
 # a list of files
diff --git a/doxygen/siphon-process b/doxygen/siphon-process
index 698da8828e1..411bf72f0f0 100755
--- a/doxygen/siphon-process
+++ b/doxygen/siphon-process
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2016 Comcast Cable Communications Management, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,39 +17,45 @@
 # The idea is to siphon off certain initializers so that we can better
 # auto-document the contents of that initializer.
 
-import os, sys, argparse, logging
+import argparse
+import logging
+import os
+import sys
+
 import siphon
 
 DEFAULT_LOGFILE = None
 DEFAULT_LOGLEVEL = "info"
-DEFAULT_SIPHON ="clicmd"
+DEFAULT_SIPHON = "clicmd"
 DEFAULT_FORMAT = "markdown"
 DEFAULT_OUTPUT = None
 DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
 
 ap = argparse.ArgumentParser()
 ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
-        help="Log file [%s]" % DEFAULT_LOGFILE)
+                help="Log file [%s]" % DEFAULT_LOGFILE)
 ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
-        choices=["debug", "info", "warning", "error", "critical"],
-        help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+                choices=["debug", "info", "warning", "error", "critical"],
+                help="Logging level [%s]" % DEFAULT_LOGLEVEL)
 
 ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
-        choices=siphon.process.siphons.keys(),
-        help="Siphon type to process [%s]" % DEFAULT_SIPHON)
+                choices=siphon.process.siphons.keys(),
+                help="Siphon type to process [%s]" % DEFAULT_SIPHON)
 ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
-        choices=siphon.process.formats.keys(),
-        help="Output format to generate [%s]" % DEFAULT_FORMAT)
+                choices=siphon.process.formats.keys(),
+                help="Output format to generate [%s]" % DEFAULT_FORMAT)
 ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
-        help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT)
+                help="Output file (uses stdout if not defined) [%s]" %
+                     DEFAULT_OUTPUT)
 ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
-        help="Path to render templates directory [%s]" % DEFAULT_TEMPLATES)
+                help="Path to render templates directory [%s]" %
+                     DEFAULT_TEMPLATES)
 ap.add_argument("input", nargs='+', metavar="input_file",
-        help="Input .siphon files")
+                help="Input .siphon files")
 args = ap.parse_args()
 
 logging.basicConfig(filename=args.log_file,
-        level=getattr(logging, args.log_level.upper(), None))
+                    level=getattr(logging, args.log_level.upper(), None))
 log = logging.getLogger("siphon_process")
 
 # Determine where to send the generated output
diff --git a/doxygen/siphon/__init__.py b/doxygen/siphon/__init__.py
index 437a1df1597..f6417314d82 100644
--- a/doxygen/siphon/__init__.py
+++ b/doxygen/siphon/__init__.py
@@ -14,11 +14,11 @@
 
 # Siphon classes
 
-import generate
-import generate_clicmd
-import generate_syscfg
+from . import generate
+from . import generate_clicmd
+from . import generate_syscfg
 
-import parsers
-import process
-import process_clicmd
-import process_syscfg
+from . import parsers
+from . import process
+from . import process_clicmd
+from . import process_syscfg
diff --git a/doxygen/siphon/generate.py b/doxygen/siphon/generate.py
index d6b6faf34f7..2ae5a1b6f1b 100644
--- a/doxygen/siphon/generate.py
+++ b/doxygen/siphon/generate.py
@@ -14,12 +14,14 @@
 
 # Generate .siphon source fragments for later processing
 
+import json
 import logging
-import os, sys, re, json
+import os
+import re
 
 """List of (regexp, siphon_name) tuples for matching the start of C
    initializer blocks in source files. Each siphon class registers
-   themselves on tihs list."""
+   themselves on this list."""
 siphon_patterns = []
 
 class Generate(object):
diff --git a/doxygen/siphon/generate_clicmd.py b/doxygen/siphon/generate_clicmd.py
index 7b13111028d..6d24aaf4926 100644
--- a/doxygen/siphon/generate_clicmd.py
+++ b/doxygen/siphon/generate_clicmd.py
@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import generate, re
+import re
+
+from . import generate
 
 # Register our regexp
 generate.siphon_patterns.append((
diff --git a/doxygen/siphon/generate_syscfg.py b/doxygen/siphon/generate_syscfg.py
index c77936a9dc8..52c802e5752 100644
--- a/doxygen/siphon/generate_syscfg.py
+++ b/doxygen/siphon/generate_syscfg.py
@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import generate, re
+import re
+
+from . import generate
 
 # Register our regexp
 generate.siphon_patterns.append((
diff --git a/doxygen/siphon/process.py b/doxygen/siphon/process.py
index f3119ea89e8..57e323e48ac 100644
--- a/doxygen/siphon/process.py
+++ b/doxygen/siphon/process.py
@@ -14,7 +14,13 @@
 
 # Generation template class
 
-import logging, os,sys, cgi, json, jinja2, HTMLParser
+import html.parser
+import json
+import logging
+import os
+import sys
+
+import jinja2
 
 # Classes register themselves in this dictionary
 """Mapping of known processors to their classes"""
@@ -24,8 +30,8 @@ siphons = {}
 formats = {}
 
 
-"""Generate rendered output for siphoned data."""
 class Siphon(object):
+    """Generate rendered output for siphoned data."""
 
     # Set by subclasses
     """Our siphon name"""
@@ -79,12 +85,11 @@ class Siphon(object):
             keep_trailing_newline=True)
 
         # Convenience, get a reference to the internal escape and
-        # unescape methods in cgi and HTMLParser. These then become
+        # unescape methods in html.parser. These then become
         # available to templates to use, if needed.
-        self._h = HTMLParser.HTMLParser()
-        self.escape = cgi.escape
-        self.unescape = self._h.unescape
-
+        self._h = html.parser.HTMLParser()
+        self.escape = html.escape
+        self.unescape = html.unescape
 
     # Output renderers
 
@@ -168,12 +173,11 @@ class Siphon(object):
 
     """Template processor"""
     def template(self, name, **kwargs):
-      tpl = self._tplenv.get_template(name + self._format.extension)
-      return tpl.render(
+        tpl = self._tplenv.get_template(name + self._format.extension)
+        return tpl.render(
             this=self,
             **kwargs)
 
-
     # Processing methods
 
     """Parse the input file into a more usable dictionary structure."""
@@ -196,11 +200,11 @@ class Siphon(object):
             for item in data["items"]:
                 try:
                     o = self._parser.parse(item['block'])
-                except:
-                    self.log.error("Exception parsing item: %s\n%s" \
-                            % (json.dumps(item, separators=(',', ': '),
-                                indent=4),
-                                item['block']))
+                except Exception:
+                    self.log.error("Exception parsing item: %s\n%s"
+                                   % (json.dumps(item, separators=(',', ': '),
+                                                 indent=4),
+                                      item['block']))
                     raise
 
                 # Augment the item with metadata
@@ -247,8 +251,8 @@ class Siphon(object):
             if group.startswith('_'):
                 continue
 
-            self.log.info("Processing items in group \"%s\" (%s)." % \
-                (group, group_sort_key(group)))
+            self.log.info("Processing items in group \"%s\" (%s)." %
+                          (group, group_sort_key(group)))
 
             # Generate the section index entry (write it now)
             out.write(self.index_section(group))
@@ -260,8 +264,8 @@ class Siphon(object):
                 return self.item_sort_key(self._cmds[group][key])
 
             for key in sorted(self._cmds[group].keys(), key=item_sort_key):
-                self.log.debug("--- Processing key \"%s\" (%s)." % \
-                    (key, item_sort_key(key)))
+                self.log.debug("--- Processing key \"%s\" (%s)." %
+                               (key, item_sort_key(key)))
 
                 o = self._cmds[group][key]
                 meta = {
@@ -283,8 +287,8 @@ class Siphon(object):
         out.write(contents)
 
 
-"""Output format class"""
 class Format(object):
+    """Output format class"""
 
     """Name of this output format"""
     name = None
@@ -293,19 +297,21 @@ class Format(object):
     extension = None
 
 
-"""Markdown output format"""
 class FormatMarkdown(Format):
+    """Markdown output format"""
     name = "markdown"
     extension = ".md"
 
+
 # Register 'markdown'
 formats["markdown"] = FormatMarkdown
 
 
-"""Itemlist output format"""
 class FormatItemlist(Format):
+    """Itemlist output format"""
     name = "itemlist"
     extension = ".itemlist"
 
+
 # Register 'itemlist'
 formats["itemlist"] = FormatItemlist
diff --git a/doxygen/siphon/process_clicmd.py b/doxygen/siphon/process_clicmd.py
index 9b3bd35c86d..231c7a7cfac 100644
--- a/doxygen/siphon/process_clicmd.py
+++ b/doxygen/siphon/process_clicmd.py
@@ -14,7 +14,8 @@
 
 # Generate clicmd formatted output
 
-import process, parsers
+from . import process, parsers
+
 
 class SiphonCLICMD(process.Siphon):
 
@@ -25,7 +26,6 @@ class SiphonCLICMD(process.Siphon):
         super(SiphonCLICMD, self).__init__(*args, **kwargs)
         self._parser = parsers.MacroInitializer()
 
-
     # Output renderers
 
     def index_sort_key(self, group):
diff --git a/doxygen/siphon/process_syscfg.py b/doxygen/siphon/process_syscfg.py
index 94be591039d..bccde2c153d 100644
--- a/doxygen/siphon/process_syscfg.py
+++ b/doxygen/siphon/process_syscfg.py
@@ -14,7 +14,8 @@
 
 # Generate syscfg formatted output
 
-import process, parsers
+from . import process, parsers
+
 
 class SiphonSYSCFG(process.Siphon):