# Copyright (c) 2016 Comcast Cable Communications Management, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Generation template class import html.parser import json import logging import os import sys import re import jinja2 # Classes register themselves in this dictionary """Mapping of known processors to their classes""" siphons = {} """Mapping of known output formats to their classes""" formats = {} class Siphon(object): """Generate rendered output for siphoned data.""" # Set by subclasses """Our siphon name""" name = None # Set by subclasses """Name of an identifier used by this siphon""" identifier = None # Set by subclasses """The pyparsing object to use to parse with""" _parser = None """The input data""" _cmds = None """Group key to (directory,file) mapping""" _group = None """Logging handler""" log = None """Directory to look for siphon rendering templates""" template_directory = None """Directory to output parts in""" outdir = None """Template environment, if we're using templates""" _tplenv = None def __init__(self, template_directory, format, outdir, repository_link): super(Siphon, self).__init__() self.log = logging.getLogger("siphon.process.%s" % self.name) # Get our output format details fmt_klass = formats[format] fmt = fmt_klass() self._format = fmt # Sort out the template search path def _tpldir(name): return os.sep.join((template_directory, fmt.name, name)) self.template_directory = template_directory searchpath = [ _tpldir(self.name), _tpldir("default"), ] self.outdir = outdir loader = jinja2.FileSystemLoader(searchpath=searchpath) self._tplenv = jinja2.Environment( loader=loader, trim_blocks=True, autoescape=False, keep_trailing_newline=True) # Convenience, get a reference to the internal escape and # unescape methods in html.parser. These then become # available to templates to use, if needed. self._h = html.parser.HTMLParser() self.escape = html.escape self.unescape = html.unescape # TODO: customize release self.repository_link = repository_link # Output renderers """Returns an object to be used as the sorting key in the item index.""" def index_sort_key(self, group): return group """Returns a string to use as the header at the top of the item index.""" def index_header(self): return self.template("index_header") """Returns the string fragment to use for each section in the item index.""" def index_section(self, group): return self.template("index_section", group=group) """Returns the string fragment to use for each entry in the item index.""" def index_entry(self, meta, item): return self.template("index_entry", meta=meta, item=item) """Returns an object, typically a string, to be used as the sorting key for items within a section.""" def item_sort_key(self, item): return item['name'] """Returns a key for grouping items together.""" def group_key(self, directory, file, macro, name): _global = self._cmds['_global'] if file in _global and 'group_label' in _global[file]: self._group[file] = (directory, file) return file self._group[directory] = (directory, None) return directory """Returns a key for identifying items within a grouping.""" def item_key(self, directory, file, macro, name): return name """Returns a string to use as the header when rendering the item.""" def item_header(self, group): return self.template("item_header", group=group) """Returns a string to use as the body when rendering the item.""" def item_format(self, meta, item): return self.template("item_format", meta=meta, item=item) """Returns a string to use as the label for the page reference.""" def page_label(self, group): return "_".join(( self.name, self.sanitize_label(group) )) """Returns a title to use for a page.""" def page_title(self, group): _global = self._cmds['_global'] (directory, file) = self._group[group] if file and file in _global and 'group_label' in _global[file]: return _global[file]['group_label'] if directory in _global and 'group_label' in _global[directory]: return _global[directory]['group_label'] return directory """Returns a string to use as the label for the section reference.""" def item_label(self, group, item): return "__".join(( self.name, item )) """Label sanitizer; for creating Doxygen references""" def sanitize_label(self, value): return value.replace(" ", "_") \ .replace("/", "_") \ .replace(".", "_") """Template processor""" def template(self, name, **kwargs): tpl = self._tplenv.get_template(name + self._format.extension) return tpl.render( this=self, **kwargs) # Processing methods """Parse the input file into a more usable dictionary structure.""" def load_json(self, files): self._cmds = {} self._group = {} line_num = 0 line_start = 0 for filename in files: filename = os.path.relpath(filename) self.log.info("Parsing items in file \"%s\"." % filename) data = None with open(filename, "r") as fd: data = json.load(fd) self._cmds['_global'] = data['global'] # iterate the items loaded and regroup it for item in data["items"]: try: o = self._parser.parse(item['block']) except Exception: self.log.error("Exception parsing item: %s\n%s" % (json.dumps(item, separators=(',', ': '), indent=4), item['block'])) raise # Augment the item with metadata o["meta"] = {} for key in item: if key == 'block': continue o['meta'][key] = item[key] # Load some interesting fields directory = item['directory'] file = item['file'] macro = o["macro"] name = o["name"] # Generate keys to group items by group_key = self.group_key(directory, file, macro, name) item_key = self.item_key(directory, file, macro, name) if group_key not in self._cmds: self._cmds[group_key] = {} self._cmds[group_key][item_key] = o """Iterate over the input data, calling render methods to generate the output.""" def process(self, out=None): if out is None: out = sys.stdout # Accumulated body contents contents = "" # Write the header for this siphon type out.write(self.index_header()) # Sort key helper for the index def group_sort_key(group): return self.index_sort_key(group) # Iterate the dictionary and process it for group in sorted(self._cmds.keys(), key=group_sort_key): if group.startswith('_'): continue self.log.info("Processing items in group \"%s\" (%s)." % (group, group_sort_key(group))) # Generate the section index entry (write it now) out.write(self.index_section(group)) # Generate the item header (save for later) contents += self.item_header(group) def item_sort_key(key): return self.item_sort_key(self._cmds[group][key]) for key in sorted(self._cmds[group].keys(), key=item_sort_key): self.log.debug("--- Processing key \"%s\" (%s)." % (key, item_sort_key(key))) o = self._cmds[group][key] meta = { "directory": o['meta']['directory'], "file": o['meta']['file'], "macro": o['macro'], "name": o['name'], "key": key, "label": self.item_label(group, key), } # Generate the index entry for the item (write it now) out.write(self.index_entry(meta, o)) # Generate the item itself (save for later) contents += self.item_format(meta, o) page_name = self.separate_page_names(group) if page_name != "": path = os.path.join(self.outdir, page_name) with open(path, "w+") as page: page.write(contents) contents = "" # Deliver the accumulated body output out.write(contents) def do_cliexstart(self, matchobj): title = matchobj.group(1) title = ' '.join(title.splitlines()) content = matchobj.group(2) content = re.sub(r"\n", r"\n ", content) return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content) def do_clistart(self, matchobj): content = matchobj.group(1) content = re.sub(r"\n", r"\n ", content) return "\n\n.. code-block:: console\n\n %s\n\n" % content def do_cliexcmd(self, matchobj): content = matchobj.group(1) content = ' '.join(content.splitlines()) return "\n\n.. code-block:: console\n\n %s\n\n" % content def process_list(self, matchobj): content = matchobj.group(1) content = self.reindent(content, 2) return "@@@@%s\nBBBB" % content def process_special(self, s): # ----------- markers to remove s = re.sub(r"@cliexpar\s*", r"", s) s = re.sub(r"@parblock\s*", r"", s) s = re.sub(r"@endparblock\s*", r"", s) s = re.sub(r"<br>", "", s) # ----------- emphasis # <b><em> s = re.sub(r"<b><em>\s*", "``", s) s = re.sub(r"\s*</b></em>", "``", s) s = re.sub(r"\s*</em></b>", "``", s) # <b> s = re.sub(r"<b>\s*", "**", s) s = re.sub(r"\s*</b>", "**", s) # <code> s = re.sub(r"<code>\s*", "``", s) s = re.sub(r"\s*</code>", "``", s) # <em> s = re.sub(r"'?<em>\s*", r"``", s) s = re.sub(r"\s*</em>'?", r"``", s) # @c <something> s = re.sub(r"@c\s(\S+)", r"``\1``", s) # ----------- todos s = re.sub(r"@todo[^\n]*", "", s) s = re.sub(r"@TODO[^\n]*", "", s) # ----------- code blocks s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL) s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL) s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL) # ----------- lists s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE) s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL) s = re.sub(r"BBBB@@@@", r"-", s) s = re.sub(r"@@@@", r"-", s) s = re.sub(r"BBBB", r"\n\n", s) # ----------- Cleanup remains s = re.sub(r"@cliexend\s*", r"", s) return s def separate_page_names(self, group): return "" # This push the given textblock <indent> spaces right def reindent(self, s, indent): ind = " " * indent s = re.sub(r"\n", "\n" + ind, s) return s # This aligns the given textblock left (no indent) def noindent(self, s): s = re.sub(r"\n[ \f\v\t]*", "\n", s) return s class Format(object): """Output format class""" """Name of this output format""" name = None """Expected file extension of templates that build this format""" extension = None class FormatMarkdown(Format): """Markdown output format""" name = "markdown" extension = ".md" # Register 'markdown' formats["markdown"] = FormatMarkdown class FormatItemlist(Format): """Itemlist output format""" name = "itemlist" extension = ".itemlist" # Register 'itemlist' formats["itemlist"] = FormatItemlist