summaryrefslogtreecommitdiffstats
path: root/doxygen/siphon/generate.py
diff options
context:
space:
mode:
authorChris Luke <chrisy@flirble.org>2016-09-12 08:55:13 -0400
committerChris Luke <chrisy@flirble.org>2016-09-21 15:42:25 -0400
commit90f52bf990791ea73479ffc50fc1eb3450de443a (patch)
treeffcd6f5a94c4fc1a44ce9a3e088f18449007e3db /doxygen/siphon/generate.py
parentce64b8e5b247149887caf77fd139d2a6880acbe6 (diff)
Refactor pre-Doxy siphon scripts; VPP-396
- Modularize the code to make the Siphon process easier to maintain. - Move much of the output rendering into Jinja2 templates. - Add syscfg siphon type for startup config documentation. - Add sample syscfg documentation. - Add clicfg and syscfg preamble docs, adapted from their wiki pages. - Fix sorting of CLI items across multiple directories. Change-Id: Ib8288fe005adfea68ceed75a38ff8eba25d3cc79 Signed-off-by: Chris Luke <chrisy@flirble.org>
Diffstat (limited to 'doxygen/siphon/generate.py')
-rw-r--r--doxygen/siphon/generate.py304
1 files changed, 304 insertions, 0 deletions
diff --git a/doxygen/siphon/generate.py b/doxygen/siphon/generate.py
new file mode 100644
index 00000000000..d6b6faf34f7
--- /dev/null
+++ b/doxygen/siphon/generate.py
@@ -0,0 +1,304 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate .siphon source fragments for later processing
+
+import logging
+import os, sys, re, json
+
+"""List of (regexp, siphon_name) tuples for matching the start of C
+ initializer blocks in source files. Each siphon class registers
+ themselves on tihs list."""
+siphon_patterns = []
+
+class Generate(object):
+ """Matches a siphon comment block start"""
+ siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
+
+ """Matches a siphon comment block stop"""
+ siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
+
+ """Siphon block directive delimiter"""
+ siphon_block_delimiter = "%%"
+
+ """Matches a siphon block directive such as
+ '%clicmd:group_label Debug CLI%'"""
+ siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
+ (siphon_block_delimiter, siphon_block_delimiter))
+
+ """Matches the start of an initializer block"""
+ siphon_initializer = re.compile("\s*=")
+
+ """Collated output for each siphon"""
+ output = None
+
+ """Directory prefix to strip from input filenames to keep things tidy."""
+ input_prefix = None
+
+ """List of known siphons"""
+ known_siphons = None
+
+ """Logging handler"""
+ log = None
+
+
+ def __init__(self, output_directory, input_prefix):
+ super(Generate, self).__init__()
+ self.log = logging.getLogger("siphon.generate")
+
+ # Build a list of known siphons
+ self.known_siphons = []
+ for item in siphon_patterns:
+ siphon = item[1]
+ if siphon not in self.known_siphons:
+ self.known_siphons.append(siphon)
+
+ # Setup information for siphons we know about
+ self.output = {}
+ for siphon in self.known_siphons:
+ self.output[siphon] = {
+ "file": "%s/%s.siphon" % (output_directory, siphon),
+ "global": {},
+ "items": [],
+ }
+
+ self.input_prefix = input_prefix
+
+
+ """
+ count open and close braces in str
+ return (0, index) when braces were found and count becomes 0.
+ index indicates the position at which the last closing brace was
+ found.
+ return (-1, -1) if a closing brace is found before any opening one.
+ return (count, -1) if not all opening braces are closed, count is the
+ current depth
+ """
+ def count_braces(self, str, count=0, found=False):
+ for index in range(0, len(str)):
+ if str[index] == '{':
+ count += 1;
+ found = True
+ elif str[index] == '}':
+ if count == 0:
+ # means we never found an open brace
+ return (-1, -1)
+ count -= 1;
+
+ if count == 0 and found:
+ return (count, index)
+
+ return (count, -1)
+
+ def parse(self, filename):
+ # Strip the current directory off the start of the
+ # filename for brevity
+ if filename[0:len(self.input_prefix)] == self.input_prefix:
+ filename = filename[len(self.input_prefix):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+ # Work out the abbreviated directory name
+ directory = os.path.dirname(filename)
+ if directory[0:2] == "./":
+ directory = directory[2:]
+ elif directory[0:len(self.input_prefix)] == self.input_prefix:
+ directory = directory[len(self.input_prefix):]
+ if directory[0] == "/":
+ directory = directory[1:]
+
+ # Open the file and explore its contents...
+ self.log.info("Siphoning from %s." % filename)
+ directives = {}
+ with open(filename) as fd:
+ siphon = None
+ close_siphon = None
+ siphon_block = ""
+ in_block = False
+ line_num = 0
+ siphon_line = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ """See if there is a block directive and if so extract it"""
+ def process_block_directive(str, directives):
+ m = self.siphon_block_directive.search(str)
+ if m is not None:
+ k = m.group(2)
+ v = m.group(3).strip()
+ directives[k] = v
+ # Return only the parts we did not match
+ return str[0:m.start(1)] + str[m.end(4):]
+
+ return str
+
+ def process_block_prefix(str):
+ if str.startswith(" * "):
+ str = str[3:]
+ elif str == " *":
+ str = ""
+ return str
+
+ if not in_block:
+ # See if the line contains the start of a siphon doc block
+ m = self.siphon_block_start.search(str)
+ if m is not None:
+ in_block = True
+ t = m.group(1)
+
+ # Now check if the block closes on the same line
+ m = self.siphon_block_stop.search(t)
+ if m is not None:
+ t = m.group(1)
+ in_block = False
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t
+
+ # Skip to next line
+ continue
+
+ else:
+ # Check to see if we have an end block marker
+ m = self.siphon_block_stop.search(str)
+ if m is not None:
+ in_block = False
+ t = m.group(1)
+ else:
+ t = str
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t + "\n"
+
+ # Skip to next line
+ continue
+
+
+ if siphon is None:
+ # Look for blocks we need to siphon
+ for p in siphon_patterns:
+ if p[0].match(str):
+ siphon = [ p[1], str + "\n", 0 ]
+ siphon_line = line_num
+
+ # see if we have an initializer
+ m = self.siphon_initializer.search(str)
+ if m is not None:
+ # count the braces on this line
+ (count, index) = \
+ self.count_braces(str[m.start():])
+ siphon[2] = count
+ # TODO - it's possible we have the
+ # initializer all on the first line
+ # we should check for it, but also
+ # account for the possibility that
+ # the open brace is on the next line
+ #if count == 0:
+ # # braces balanced
+ # close_siphon = siphon
+ # siphon = None
+ else:
+ # no initializer: close the siphon right now
+ close_siphon = siphon
+ siphon = None
+ else:
+ # See if we should end the siphon here - do we have
+ # balanced braces?
+ (count, index) = self.count_braces(str,
+ count=siphon[2], found=True)
+ if count == 0:
+ # braces balanced - add the substring and
+ # close the siphon
+ siphon[1] += str[:index+1] + ";\n"
+ close_siphon = siphon
+ siphon = None
+ else:
+ # add the whole string, move on
+ siphon[2] = count
+ siphon[1] += str + "\n"
+
+ if close_siphon is not None:
+ # Write the siphoned contents to the right place
+ siphon_name = close_siphon[0]
+
+ # Copy directives for the file
+ details = {}
+ for key in directives:
+ if ":" in key:
+ (sn, label) = key.split(":")
+ if sn == siphon_name:
+ details[label] = directives[key]
+ else:
+ details[key] = directives[key]
+
+ # Copy details for this block
+ details['file'] = filename
+ details['directory'] = directory
+ details['line_start'] = siphon_line
+ details['line_end'] = line_num
+ details['siphon_block'] = siphon_block.strip()
+ details["block"] = close_siphon[1]
+
+ # Store the item
+ self.output[siphon_name]['items'].append(details)
+
+ # All done
+ close_siphon = None
+ siphon_block = ""
+
+ # Update globals
+ for key in directives.keys():
+ if ':' not in key:
+ continue
+
+ if filename.endswith("/dir.dox"):
+ # very special! use the parent directory name
+ l = directory
+ else:
+ l = filename
+
+ (sn, label) = key.split(":")
+
+ if sn not in self.output:
+ self.output[sn] = {}
+ if 'global' not in self.output[sn]:
+ self.output[sn]['global'] = {}
+ if l not in self.output[sn]['global']:
+ self.output[sn]['global'][l] = {}
+
+ self.output[sn]['global'][l][label] = directives[key]
+
+ def deliver(self):
+ # Write out the data
+ for siphon in self.output.keys():
+ self.log.info("Saving siphon data %s." % siphon)
+ s = self.output[siphon]
+ with open(s['file'], "a") as fp:
+ json.dump(s, fp,
+ separators=(',', ': '), indent=4, sort_keys=True)
+