aboutsummaryrefslogtreecommitdiffstats
path: root/docs/_scripts/siphon
diff options
context:
space:
mode:
Diffstat (limited to 'docs/_scripts/siphon')
-rw-r--r--docs/_scripts/siphon/generate.py83
-rw-r--r--docs/_scripts/siphon/generate_clicmd.py13
-rw-r--r--docs/_scripts/siphon/generate_syscfg.py14
-rw-r--r--docs/_scripts/siphon/parsers.py52
-rw-r--r--docs/_scripts/siphon/process.py109
-rw-r--r--docs/_scripts/siphon/process_clicmd.py30
6 files changed, 169 insertions, 132 deletions
diff --git a/docs/_scripts/siphon/generate.py b/docs/_scripts/siphon/generate.py
index 2ae5a1b6f1b..1244c4658e4 100644
--- a/docs/_scripts/siphon/generate.py
+++ b/docs/_scripts/siphon/generate.py
@@ -24,8 +24,10 @@ import re
themselves on this list."""
siphon_patterns = []
+
class Generate(object):
"""Matches a siphon comment block start"""
+
siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
"""Matches a siphon comment block stop"""
@@ -36,8 +38,10 @@ class Generate(object):
"""Matches a siphon block directive such as
'%clicmd:group_label Debug CLI%'"""
- siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
- (siphon_block_delimiter, siphon_block_delimiter))
+ siphon_block_directive = re.compile(
+ "(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)"
+ % (siphon_block_delimiter, siphon_block_delimiter)
+ )
"""Matches the start of an initializer block"""
siphon_initializer = re.compile("\s*=")
@@ -54,7 +58,6 @@ class Generate(object):
"""Logging handler"""
log = None
-
def __init__(self, output_directory, input_prefix):
super(Generate, self).__init__()
self.log = logging.getLogger("siphon.generate")
@@ -70,14 +73,13 @@ class Generate(object):
self.output = {}
for siphon in self.known_siphons:
self.output[siphon] = {
- "file": "%s/%s.siphon" % (output_directory, siphon),
- "global": {},
- "items": [],
- }
+ "file": "%s/%s.siphon" % (output_directory, siphon),
+ "global": {},
+ "items": [],
+ }
self.input_prefix = input_prefix
-
"""
count open and close braces in str
return (0, index) when braces were found and count becomes 0.
@@ -87,16 +89,17 @@ class Generate(object):
return (count, -1) if not all opening braces are closed, count is the
current depth
"""
+
def count_braces(self, str, count=0, found=False):
for index in range(0, len(str)):
- if str[index] == '{':
- count += 1;
+ if str[index] == "{":
+ count += 1
found = True
- elif str[index] == '}':
+ elif str[index] == "}":
if count == 0:
# means we never found an open brace
return (-1, -1)
- count -= 1;
+ count -= 1
if count == 0 and found:
return (count, index)
@@ -106,8 +109,8 @@ class Generate(object):
def parse(self, filename):
# Strip the current directory off the start of the
# filename for brevity
- if filename[0:len(self.input_prefix)] == self.input_prefix:
- filename = filename[len(self.input_prefix):]
+ if filename[0 : len(self.input_prefix)] == self.input_prefix:
+ filename = filename[len(self.input_prefix) :]
if filename[0] == "/":
filename = filename[1:]
@@ -115,8 +118,8 @@ class Generate(object):
directory = os.path.dirname(filename)
if directory[0:2] == "./":
directory = directory[2:]
- elif directory[0:len(self.input_prefix)] == self.input_prefix:
- directory = directory[len(self.input_prefix):]
+ elif directory[0 : len(self.input_prefix)] == self.input_prefix:
+ directory = directory[len(self.input_prefix) :]
if directory[0] == "/":
directory = directory[1:]
@@ -133,9 +136,10 @@ class Generate(object):
for line in fd:
line_num += 1
- str = line[:-1] # filter \n
+ str = line[:-1] # filter \n
"""See if there is a block directive and if so extract it"""
+
def process_block_directive(str, directives):
m = self.siphon_block_directive.search(str)
if m is not None:
@@ -143,7 +147,7 @@ class Generate(object):
v = m.group(3).strip()
directives[k] = v
# Return only the parts we did not match
- return str[0:m.start(1)] + str[m.end(4):]
+ return str[0 : m.start(1)] + str[m.end(4) :]
return str
@@ -200,27 +204,25 @@ class Generate(object):
# Skip to next line
continue
-
if siphon is None:
# Look for blocks we need to siphon
for p in siphon_patterns:
if p[0].match(str):
- siphon = [ p[1], str + "\n", 0 ]
+ siphon = [p[1], str + "\n", 0]
siphon_line = line_num
# see if we have an initializer
m = self.siphon_initializer.search(str)
if m is not None:
# count the braces on this line
- (count, index) = \
- self.count_braces(str[m.start():])
+ (count, index) = self.count_braces(str[m.start() :])
siphon[2] = count
# TODO - it's possible we have the
# initializer all on the first line
# we should check for it, but also
# account for the possibility that
# the open brace is on the next line
- #if count == 0:
+ # if count == 0:
# # braces balanced
# close_siphon = siphon
# siphon = None
@@ -231,12 +233,11 @@ class Generate(object):
else:
# See if we should end the siphon here - do we have
# balanced braces?
- (count, index) = self.count_braces(str,
- count=siphon[2], found=True)
+ (count, index) = self.count_braces(str, count=siphon[2], found=True)
if count == 0:
# braces balanced - add the substring and
# close the siphon
- siphon[1] += str[:index+1] + ";\n"
+ siphon[1] += str[: index + 1] + ";\n"
close_siphon = siphon
siphon = None
else:
@@ -259,15 +260,15 @@ class Generate(object):
details[key] = directives[key]
# Copy details for this block
- details['file'] = filename
- details['directory'] = directory
- details['line_start'] = siphon_line
- details['line_end'] = line_num
- details['siphon_block'] = siphon_block.strip()
+ details["file"] = filename
+ details["directory"] = directory
+ details["line_start"] = siphon_line
+ details["line_end"] = line_num
+ details["siphon_block"] = siphon_block.strip()
details["block"] = close_siphon[1]
# Store the item
- self.output[siphon_name]['items'].append(details)
+ self.output[siphon_name]["items"].append(details)
# All done
close_siphon = None
@@ -275,7 +276,7 @@ class Generate(object):
# Update globals
for key in directives.keys():
- if ':' not in key:
+ if ":" not in key:
continue
if filename.endswith("/dir.dox"):
@@ -288,19 +289,17 @@ class Generate(object):
if sn not in self.output:
self.output[sn] = {}
- if 'global' not in self.output[sn]:
- self.output[sn]['global'] = {}
- if l not in self.output[sn]['global']:
- self.output[sn]['global'][l] = {}
+ if "global" not in self.output[sn]:
+ self.output[sn]["global"] = {}
+ if l not in self.output[sn]["global"]:
+ self.output[sn]["global"][l] = {}
- self.output[sn]['global'][l][label] = directives[key]
+ self.output[sn]["global"][l][label] = directives[key]
def deliver(self):
# Write out the data
for siphon in self.output.keys():
self.log.info("Saving siphon data %s." % siphon)
s = self.output[siphon]
- with open(s['file'], "a") as fp:
- json.dump(s, fp,
- separators=(',', ': '), indent=4, sort_keys=True)
-
+ with open(s["file"], "a") as fp:
+ json.dump(s, fp, separators=(",", ": "), indent=4, sort_keys=True)
diff --git a/docs/_scripts/siphon/generate_clicmd.py b/docs/_scripts/siphon/generate_clicmd.py
index 6d24aaf4926..2e2f6281a39 100644
--- a/docs/_scripts/siphon/generate_clicmd.py
+++ b/docs/_scripts/siphon/generate_clicmd.py
@@ -17,8 +17,11 @@ import re
from . import generate
# Register our regexp
-generate.siphon_patterns.append((
- re.compile("(?P<m>VLIB_CLI_COMMAND)\s*"
- "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
- "clicmd"
-))
+generate.siphon_patterns.append(
+ (
+ re.compile(
+ "(?P<m>VLIB_CLI_COMMAND)\s*" "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"
+ ),
+ "clicmd",
+ )
+)
diff --git a/docs/_scripts/siphon/generate_syscfg.py b/docs/_scripts/siphon/generate_syscfg.py
index 52c802e5752..105a59c8262 100644
--- a/docs/_scripts/siphon/generate_syscfg.py
+++ b/docs/_scripts/siphon/generate_syscfg.py
@@ -17,8 +17,12 @@ import re
from . import generate
# Register our regexp
-generate.siphon_patterns.append((
- re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*"
- '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'),
- "syscfg"
-))
+generate.siphon_patterns.append(
+ (
+ re.compile(
+ "(?P<m>VLIB_CONFIG_FUNCTION)\s*"
+ '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'
+ ),
+ "syscfg",
+ )
+)
diff --git a/docs/_scripts/siphon/parsers.py b/docs/_scripts/siphon/parsers.py
index 162205de4ca..1a7d1f59539 100644
--- a/docs/_scripts/siphon/parsers.py
+++ b/docs/_scripts/siphon/parsers.py
@@ -18,9 +18,10 @@ ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
intNum = pp.Word(pp.nums)
hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
octalNum = pp.Literal("0") + pp.Word("01234567")
-integer = (hexNum | octalNum | intNum) + \
- pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
-floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
+integer = (hexNum | octalNum | intNum) + pp.Optional(
+ pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L")
+)
+floatNum = pp.Regex(r"\d+(\.\d*)?([eE]\d+)?") + pp.Optional(pp.Literal("f"))
char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
arrayIndex = integer | ident
@@ -36,23 +37,29 @@ semicolon = pp.Literal(";").suppress()
# initializer := { [member = ] (variable | expression | { initializer } ) }
typeName = ident
varName = ident
-typeSpec = pp.Optional("unsigned") + \
- pp.oneOf("int long short float double char u8 i8 void") + \
- pp.Optional(pp.Word("*"), default="")
-typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
-
-string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
- escChar='\\', multiline=True)), adjacent=False)
+typeSpec = (
+ pp.Optional("unsigned")
+ + pp.oneOf("int long short float double char u8 i8 void")
+ + pp.Optional(pp.Word("*"), default="")
+)
+typeCast = pp.Combine("(" + (typeSpec | typeName) + ")").suppress()
+
+string = pp.Combine(
+ pp.OneOrMore(pp.QuotedString(quoteChar='"', escChar="\\", multiline=True)),
+ adjacent=False,
+)
literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
-var = pp.Combine(pp.Optional(typeCast) + varName +
- pp.Optional("[" + arrayIndex + "]"))
+var = pp.Combine(pp.Optional(typeCast) + varName + pp.Optional("[" + arrayIndex + "]"))
# This could be more complete, but suffices for our uses
-expr = (literal | var)
+expr = literal | var
"""Parse and render a block of text into a Python dictionary."""
+
+
class Parser(object):
"""Compiled PyParsing BNF"""
+
_parser = None
def __init__(self):
@@ -71,6 +78,8 @@ class Parser(object):
"""Parser for function-like macros - without the closing semi-colon."""
+
+
class ParserFunctionMacro(Parser):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix")
@@ -91,6 +100,8 @@ class ParserFunctionMacro(Parser):
"""Parser for function-like macros with a closing semi-colon."""
+
+
class ParseFunctionMacroStmt(ParserFunctionMacro):
def BNF(self):
# VLIB_CONFIG_FUNCTION (unix_config, "unix");
@@ -106,6 +117,8 @@ Parser for our struct initializers which are composed from a
function-like macro, equals sign, and then a normal C struct initializer
block.
"""
+
+
class MacroInitializer(ParserFunctionMacro):
def BNF(self):
# VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
@@ -115,14 +128,15 @@ class MacroInitializer(ParserFunctionMacro):
# };
cs = pp.Forward()
-
- member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"),
- adjacent=False)
- value = (expr | cs)
+ member = pp.Combine(
+ dot + varName + pp.Optional("[" + arrayIndex + "]"), adjacent=False
+ )
+ value = expr | cs
entry = pp.Group(pp.Optional(member + equals, default="") + value)
- entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
- (pp.ZeroOrMore(entry + comma))
+ entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | (
+ pp.ZeroOrMore(entry + comma)
+ )
cs << (lbrace + entries + rbrace)
diff --git a/docs/_scripts/siphon/process.py b/docs/_scripts/siphon/process.py
index e3a70152487..341b7cba299 100644
--- a/docs/_scripts/siphon/process.py
+++ b/docs/_scripts/siphon/process.py
@@ -88,7 +88,8 @@ class Siphon(object):
loader=loader,
trim_blocks=True,
autoescape=False,
- keep_trailing_newline=True)
+ keep_trailing_newline=True,
+ )
# Convenience, get a reference to the internal escape and
# unescape methods in html.parser. These then become
@@ -103,32 +104,38 @@ class Siphon(object):
# Output renderers
"""Returns an object to be used as the sorting key in the item index."""
+
def index_sort_key(self, group):
return group
"""Returns a string to use as the header at the top of the item index."""
+
def index_header(self):
return self.template("index_header")
"""Returns the string fragment to use for each section in the item
index."""
+
def index_section(self, group):
return self.template("index_section", group=group)
"""Returns the string fragment to use for each entry in the item index."""
+
def index_entry(self, meta, item):
return self.template("index_entry", meta=meta, item=item)
"""Returns an object, typically a string, to be used as the sorting key
for items within a section."""
+
def item_sort_key(self, item):
- return item['name']
+ return item["name"]
"""Returns a key for grouping items together."""
+
def group_key(self, directory, file, macro, name):
- _global = self._cmds['_global']
+ _global = self._cmds["_global"]
- if file in _global and 'group_label' in _global[file]:
+ if file in _global and "group_label" in _global[file]:
self._group[file] = (directory, file)
return file
@@ -136,60 +143,59 @@ class Siphon(object):
return directory
"""Returns a key for identifying items within a grouping."""
+
def item_key(self, directory, file, macro, name):
return name
"""Returns a string to use as the header when rendering the item."""
+
def item_header(self, group):
return self.template("item_header", group=group)
"""Returns a string to use as the body when rendering the item."""
+
def item_format(self, meta, item):
return self.template("item_format", meta=meta, item=item)
"""Returns a string to use as the label for the page reference."""
+
def page_label(self, group):
- return "_".join((
- self.name,
- self.sanitize_label(group)
- ))
+ return "_".join((self.name, self.sanitize_label(group)))
"""Returns a title to use for a page."""
+
def page_title(self, group):
- _global = self._cmds['_global']
+ _global = self._cmds["_global"]
(directory, file) = self._group[group]
- if file and file in _global and 'group_label' in _global[file]:
- return _global[file]['group_label']
+ if file and file in _global and "group_label" in _global[file]:
+ return _global[file]["group_label"]
- if directory in _global and 'group_label' in _global[directory]:
- return _global[directory]['group_label']
+ if directory in _global and "group_label" in _global[directory]:
+ return _global[directory]["group_label"]
return directory
"""Returns a string to use as the label for the section reference."""
+
def item_label(self, group, item):
- return "__".join((
- self.name,
- item
- ))
+ return "__".join((self.name, item))
"""Label sanitizer; for creating Doxygen references"""
+
def sanitize_label(self, value):
- return value.replace(" ", "_") \
- .replace("/", "_") \
- .replace(".", "_")
+ return value.replace(" ", "_").replace("/", "_").replace(".", "_")
"""Template processor"""
+
def template(self, name, **kwargs):
tpl = self._tplenv.get_template(name + self._format.extension)
- return tpl.render(
- this=self,
- **kwargs)
+ return tpl.render(this=self, **kwargs)
# Processing methods
"""Parse the input file into a more usable dictionary structure."""
+
def load_json(self, files):
self._cmds = {}
self._group = {}
@@ -198,34 +204,37 @@ class Siphon(object):
line_start = 0
for filename in files:
filename = os.path.relpath(filename)
- self.log.info("Parsing items in file \"%s\"." % filename)
+ self.log.info('Parsing items in file "%s".' % filename)
data = None
with open(filename, "r") as fd:
data = json.load(fd)
- self._cmds['_global'] = data['global']
+ self._cmds["_global"] = data["global"]
# iterate the items loaded and regroup it
for item in data["items"]:
try:
- o = self._parser.parse(item['block'])
+ o = self._parser.parse(item["block"])
except Exception:
- self.log.error("Exception parsing item: %s\n%s"
- % (json.dumps(item, separators=(',', ': '),
- indent=4),
- item['block']))
+ self.log.error(
+ "Exception parsing item: %s\n%s"
+ % (
+ json.dumps(item, separators=(",", ": "), indent=4),
+ item["block"],
+ )
+ )
raise
# Augment the item with metadata
o["meta"] = {}
for key in item:
- if key == 'block':
+ if key == "block":
continue
- o['meta'][key] = item[key]
+ o["meta"][key] = item[key]
# Load some interesting fields
- directory = item['directory']
- file = item['file']
+ directory = item["directory"]
+ file = item["file"]
macro = o["macro"]
name = o["name"]
@@ -240,6 +249,7 @@ class Siphon(object):
"""Iterate over the input data, calling render methods to generate the
output."""
+
def process(self, out=None):
if out is None:
@@ -257,11 +267,12 @@ class Siphon(object):
# Iterate the dictionary and process it
for group in sorted(self._cmds.keys(), key=group_sort_key):
- if group.startswith('_'):
+ if group.startswith("_"):
continue
- self.log.info("Processing items in group \"%s\" (%s)." %
- (group, group_sort_key(group)))
+ self.log.info(
+ 'Processing items in group "%s" (%s).' % (group, group_sort_key(group))
+ )
# Generate the section index entry (write it now)
out.write(self.index_section(group))
@@ -273,15 +284,16 @@ class Siphon(object):
return self.item_sort_key(self._cmds[group][key])
for key in sorted(self._cmds[group].keys(), key=item_sort_key):
- self.log.debug("--- Processing key \"%s\" (%s)." %
- (key, item_sort_key(key)))
+ self.log.debug(
+ '--- Processing key "%s" (%s).' % (key, item_sort_key(key))
+ )
o = self._cmds[group][key]
meta = {
- "directory": o['meta']['directory'],
- "file": o['meta']['file'],
- "macro": o['macro'],
- "name": o['name'],
+ "directory": o["meta"]["directory"],
+ "file": o["meta"]["file"],
+ "macro": o["macro"],
+ "name": o["name"],
"key": key,
"label": self.item_label(group, key),
}
@@ -304,7 +316,7 @@ class Siphon(object):
def do_cliexstart(self, matchobj):
title = matchobj.group(1)
- title = ' '.join(title.splitlines())
+ title = " ".join(title.splitlines())
content = matchobj.group(2)
content = re.sub(r"\n", r"\n ", content)
return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
@@ -316,7 +328,7 @@ class Siphon(object):
def do_cliexcmd(self, matchobj):
content = matchobj.group(1)
- content = ' '.join(content.splitlines())
+ content = " ".join(content.splitlines())
return "\n\n.. code-block:: console\n\n %s\n\n" % content
def process_list(self, matchobj):
@@ -351,7 +363,9 @@ class Siphon(object):
s = re.sub(r"@TODO[^\n]*", "", s)
# ----------- code blocks
s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
- s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL)
+ s = re.sub(
+ r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL
+ )
s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
# ----------- lists
s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
@@ -377,6 +391,7 @@ class Siphon(object):
s = re.sub(r"\n[ \f\v\t]*", "\n", s)
return s
+
class Format(object):
"""Output format class"""
@@ -389,6 +404,7 @@ class Format(object):
class FormatMarkdown(Format):
"""Markdown output format"""
+
name = "markdown"
extension = ".md"
@@ -399,6 +415,7 @@ formats["markdown"] = FormatMarkdown
class FormatItemlist(Format):
"""Itemlist output format"""
+
name = "itemlist"
extension = ".itemlist"
diff --git a/docs/_scripts/siphon/process_clicmd.py b/docs/_scripts/siphon/process_clicmd.py
index bf270518ad1..afc24ae6da1 100644
--- a/docs/_scripts/siphon/process_clicmd.py
+++ b/docs/_scripts/siphon/process_clicmd.py
@@ -17,6 +17,7 @@
from . import process, parsers
import os
+
class SiphonCLICMD(process.Siphon):
name = "clicmd"
@@ -32,37 +33,36 @@ class SiphonCLICMD(process.Siphon):
return self.page_label(group) + ".rst"
def index_sort_key(self, group):
- _global = self._cmds['_global']
+ _global = self._cmds["_global"]
if group not in self._group:
return group
(directory, file) = self._group[group]
- if file in _global and 'group_label' in _global[file]:
- return _global[file]['group_label']
+ if file in _global and "group_label" in _global[file]:
+ return _global[file]["group_label"]
- if directory in _global and 'group_label' in _global[directory]:
- return _global[directory]['group_label']
+ if directory in _global and "group_label" in _global[directory]:
+ return _global[directory]["group_label"]
return group
def item_sort_key(self, item):
- return item['value']['path']
+ return item["value"]["path"]
def item_label(self, group, item):
- return "_".join((
- self.name,
- self.sanitize_label(self._cmds[group][item]['value']['path'])
- ))
+ return "_".join(
+ (self.name, self.sanitize_label(self._cmds[group][item]["value"]["path"]))
+ )
def page_title(self, group):
- _global = self._cmds['_global']
+ _global = self._cmds["_global"]
(directory, file) = self._group[group]
- if file and file in _global and 'group_label' in _global[file]:
- return _global[file]['group_label']
+ if file and file in _global and "group_label" in _global[file]:
+ return _global[file]["group_label"]
- if directory in _global and 'group_label' in _global[directory]:
- return _global[directory]['group_label']
+ if directory in _global and "group_label" in _global[directory]:
+ return _global[directory]["group_label"]
file_ext = os.path.basename(directory)
fname, ext = os.path.splitext(file_ext)