Add support for using documentation siphons in multiple ways

Experiental support for generating  multiple output formats from the
same siphoned data.

Adds a contrived example to generate a plain list of all CLI commands
(the "itemlist" format).

Eventually we can consider moving the tempate procesisng into the
Output class as well as a way to override how the data is traversed
(ordered).

Change-Id: I77629a74a8fa0c7e583993469dc50491f72f13e7
Signed-off-by: Chris Luke <chrisy@flirble.org>

19 files changed, 199 insertions, 32 deletions

diff --git a/doxygen/Makefile b/doxygen/Makefile
index 7031e84d523..92fa36352fe 100644
--- a/doxygen/Makefile
+++ b/doxygen/Makefile
@@ -121,6 +121,7 @@ SIPHONS ?= clicmd syscfg
 
 SIPHON_FILES = $(addprefix $(SIPHON_INPUT)/,$(addsuffix .siphon,$(SIPHONS)))
 SIPHON_DOCS = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .md,$(SIPHONS)))
+SIPHON_ITEMLIST = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .itemlist,$(filter clicmd,$(SIPHONS))))
 
 $(BR)/.doxygen-bootstrap.ok: Makefile
 	@echo "Checking whether dependencies for Doxygen are installed..."
@@ -200,24 +201,37 @@ $(SIPHON_FILES): $(BR)/.doxygen-bootstrap.ok \
 		--output="$(SIPHON_INPUT)" \
 		"@$(SIPHON_INPUT)/files"
 
-# Process the .siphon source fragments and render them into doxygen flavored
-# markdown documentation
-.DELETE_ON_ERROR: $(SIPHON_DOCS)
-$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon \
+# Evaluate this to build a siphon doc output target for each desired
+# output type:
+# $(eval $(call siphon-process,file_extension,output_type_name))
+define siphon-process
+$(SIPHON_OUTPUT)/%.$(1): $(SIPHON_INPUT)/%.siphon \
 		$(DOXY_DIR)/siphon-process \
 		$(wildcard $(DOXY_DIR)/siphon/*.py) \
-		$(wildcard $(DOXY_DIR)/siphon_templates/*/*.md)
-	@echo "Processing siphon from $(notdir $<)..."
+		$(wildcard $(DOXY_DIR)/siphon_templates/$(2)/*/*.$(1))
+	@echo "Processing siphon for $(2) from $$(notdir $$<)..."
 	@set -e; \
 	cd "$(WS_ROOT)"; \
 	$(DOXY_DIR)/siphon-process \
-		--type=$(basename $(notdir $<)) \
-		--output="$@" \
-		"$<"
+		--type=$$(basename $$(notdir $$<)) \
+		--format=$(2) \
+		--output="$$@" \
+		"$$<"
+endef
+
+# Process the .siphon source fragments and render them into doxygen flavored
+# markdown documentation
+.DELETE_ON_ERROR: $(SIPHON_DOCS)
+$(eval $(call siphon-process,md,markdown))
+
+# Process the .siphon source fragments and render them into a list of cli
+# commands.
+.DELETE_ON_ERROR: $(SIPHON_ITEMLIST)
+$(eval $(call siphon-process,itemlist,itemlist))
 
-# This target can be used just to generate the siphoned docs
+# This target can be used just to generate the siphoned things
 .PHONY: doxygen-siphon
-doxygen-siphon: $(SIPHON_DOCS)
+doxygen-siphon: $(SIPHON_DOCS) $(SIPHON_ITEMLIST)
 
 # Generate the doxygen docs
 .PHONY: doxygen
diff --git a/doxygen/siphon-process b/doxygen/siphon-process
index ea9df96fe26..698da8828e1 100755
--- a/doxygen/siphon-process
+++ b/doxygen/siphon-process
@@ -23,6 +23,7 @@ import siphon
 DEFAULT_LOGFILE = None
 DEFAULT_LOGLEVEL = "info"
 DEFAULT_SIPHON ="clicmd"
+DEFAULT_FORMAT = "markdown"
 DEFAULT_OUTPUT = None
 DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
 
@@ -36,6 +37,9 @@ ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
 ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
         choices=siphon.process.siphons.keys(),
         help="Siphon type to process [%s]" % DEFAULT_SIPHON)
+ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
+        choices=siphon.process.formats.keys(),
+        help="Output format to generate [%s]" % DEFAULT_FORMAT)
 ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
         help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT)
 ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
@@ -56,7 +60,7 @@ else:
 
 # Get our processor
 klass = siphon.process.siphons[args.type]
-processor = klass(template_directory=args.templates)
+processor = klass(template_directory=args.templates, format=args.format)
 
 # Load the input files
 processor.load_json(args.input)
diff --git a/doxygen/siphon/process.py b/doxygen/siphon/process.py
index 34e829c5298..f3119ea89e8 100644
--- a/doxygen/siphon/process.py
+++ b/doxygen/siphon/process.py
@@ -20,6 +20,9 @@ import logging, os,sys, cgi, json, jinja2, HTMLParser
 """Mapping of known processors to their classes"""
 siphons = {}
 
+"""Mapping of known output formats to their classes"""
+formats = {}
+
 
 """Generate rendered output for siphoned data."""
 class Siphon(object):
@@ -51,28 +54,36 @@ class Siphon(object):
     """Template environment, if we're using templates"""
     _tplenv = None
 
-    def __init__(self, template_directory=None):
+    def __init__(self, template_directory, format):
         super(Siphon, self).__init__()
         self.log = logging.getLogger("siphon.process.%s" % self.name)
 
-        if template_directory is not None:
-          self.template_directory = template_directory
-          searchpath = [
-              template_directory + "/" + self.name,
-              template_directory + "/" + "default",
-          ]
-          loader = jinja2.FileSystemLoader(searchpath=searchpath)
-          self._tplenv = jinja2.Environment(
-              loader=loader,
-              trim_blocks=True,
-              keep_trailing_newline=True)
-
-          # Convenience, get a reference to the internal escape and
-          # unescape methods in cgi and HTMLParser. These then become
-          # available to templates to use, if needed.
-          self._h = HTMLParser.HTMLParser()
-          self.escape = cgi.escape
-          self.unescape = self._h.unescape
+        # Get our output format details
+        fmt_klass = formats[format]
+        fmt = fmt_klass()
+        self._format = fmt
+
+        # Sort out the template search path
+        def _tpldir(name):
+            return os.sep.join((template_directory, fmt.name, name))
+
+        self.template_directory = template_directory
+        searchpath = [
+            _tpldir(self.name),
+            _tpldir("default"),
+        ]
+        loader = jinja2.FileSystemLoader(searchpath=searchpath)
+        self._tplenv = jinja2.Environment(
+            loader=loader,
+            trim_blocks=True,
+            keep_trailing_newline=True)
+
+        # Convenience, get a reference to the internal escape and
+        # unescape methods in cgi and HTMLParser. These then become
+        # available to templates to use, if needed.
+        self._h = HTMLParser.HTMLParser()
+        self.escape = cgi.escape
+        self.unescape = self._h.unescape
 
 
     # Output renderers
@@ -157,7 +168,7 @@ class Siphon(object):
 
     """Template processor"""
     def template(self, name, **kwargs):
-      tpl = self._tplenv.get_template(name + ".md")
+      tpl = self._tplenv.get_template(name + self._format.extension)
       return tpl.render(
             this=self,
             **kwargs)
@@ -270,3 +281,31 @@ class Siphon(object):
 
         # Deliver the accumulated body output
         out.write(contents)
+
+
+"""Output format class"""
+class Format(object):
+
+    """Name of this output format"""
+    name = None
+
+    """Expected file extension of templates that build this format"""
+    extension = None
+
+
+"""Markdown output format"""
+class FormatMarkdown(Format):
+    name = "markdown"
+    extension = ".md"
+
+# Register 'markdown'
+formats["markdown"] = FormatMarkdown
+
+
+"""Itemlist output format"""
+class FormatItemlist(Format):
+    name = "itemlist"
+    extension = ".itemlist"
+
+# Register 'itemlist'
+formats["itemlist"] = FormatItemlist
diff --git a/doxygen/siphon_templates/itemlist/clicmd/item_format.itemlist b/doxygen/siphon_templates/itemlist/clicmd/item_format.itemlist
new file mode 100644
index 00000000000..195c378069a
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/clicmd/item_format.itemlist
@@ -0,0 +1,17 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{# Just output the command path #}
+{{ item['value']['path'] }}
diff --git a/doxygen/siphon_templates/itemlist/default/index_entry.itemlist b/doxygen/siphon_templates/itemlist/default/index_entry.itemlist
new file mode 100644
index 00000000000..3b2494fbd15
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/default/index_entry.itemlist
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
diff --git a/doxygen/siphon_templates/itemlist/default/index_header.itemlist b/doxygen/siphon_templates/itemlist/default/index_header.itemlist
new file mode 100644
index 00000000000..3b2494fbd15
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/default/index_header.itemlist
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
diff --git a/doxygen/siphon_templates/itemlist/default/index_section.itemlist b/doxygen/siphon_templates/itemlist/default/index_section.itemlist
new file mode 100644
index 00000000000..3b2494fbd15
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/default/index_section.itemlist
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
diff --git a/doxygen/siphon_templates/default/item_format.md b/doxygen/siphon_templates/itemlist/default/item_format.itemlist
index ed1b1bf7eeb..ed1b1bf7eeb 100644
--- a/doxygen/siphon_templates/default/item_format.md
+++ b/doxygen/siphon_templates/itemlist/default/item_format.itemlist
diff --git a/doxygen/siphon_templates/itemlist/default/item_header.itemlist b/doxygen/siphon_templates/itemlist/default/item_header.itemlist
new file mode 100644
index 00000000000..3b2494fbd15
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/default/item_header.itemlist
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
diff --git a/doxygen/siphon_templates/itemlist/syscfg/item_format.itemlist b/doxygen/siphon_templates/itemlist/syscfg/item_format.itemlist
new file mode 100644
index 00000000000..023205196f5
--- /dev/null
+++ b/doxygen/siphon_templates/itemlist/syscfg/item_format.itemlist
@@ -0,0 +1,17 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{# Just output the item name #}
+{{ item['name'] }}
diff --git a/doxygen/siphon_templates/clicmd/index_entry.md b/doxygen/siphon_templates/markdown/clicmd/index_entry.md
index 1fa9ec9b25a..1fa9ec9b25a 100644
--- a/doxygen/siphon_templates/clicmd/index_entry.md
+++ b/doxygen/siphon_templates/markdown/clicmd/index_entry.md
diff --git a/doxygen/siphon_templates/clicmd/index_header.md b/doxygen/siphon_templates/markdown/clicmd/index_header.md
index 4167f4dc477..4167f4dc477 100644
--- a/doxygen/siphon_templates/clicmd/index_header.md
+++ b/doxygen/siphon_templates/markdown/clicmd/index_header.md
diff --git a/doxygen/siphon_templates/clicmd/item_format.md b/doxygen/siphon_templates/markdown/clicmd/item_format.md
index 288dae40f24..288dae40f24 100644
--- a/doxygen/siphon_templates/clicmd/item_format.md
+++ b/doxygen/siphon_templates/markdown/clicmd/item_format.md
diff --git a/doxygen/siphon_templates/default/index_entry.md b/doxygen/siphon_templates/markdown/default/index_entry.md
index 479dcdb2cc3..479dcdb2cc3 100644
--- a/doxygen/siphon_templates/default/index_entry.md
+++ b/doxygen/siphon_templates/markdown/default/index_entry.md
diff --git a/doxygen/siphon_templates/default/index_section.md b/doxygen/siphon_templates/markdown/default/index_section.md
index 3c9d2b474c5..3c9d2b474c5 100644
--- a/doxygen/siphon_templates/default/index_section.md
+++ b/doxygen/siphon_templates/markdown/default/index_section.md
diff --git a/doxygen/siphon_templates/markdown/default/item_format.md b/doxygen/siphon_templates/markdown/default/item_format.md
new file mode 100644
index 00000000000..ed1b1bf7eeb
--- /dev/null
+++ b/doxygen/siphon_templates/markdown/default/item_format.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ raise NotImplementedError }}
diff --git a/doxygen/siphon_templates/default/item_header.md b/doxygen/siphon_templates/markdown/default/item_header.md
index 0c21e51fa5d..0c21e51fa5d 100644
--- a/doxygen/siphon_templates/default/item_header.md
+++ b/doxygen/siphon_templates/markdown/default/item_header.md
diff --git a/doxygen/siphon_templates/syscfg/index_header.md b/doxygen/siphon_templates/markdown/syscfg/index_header.md
index 5d338a0472b..5d338a0472b 100644
--- a/doxygen/siphon_templates/syscfg/index_header.md
+++ b/doxygen/siphon_templates/markdown/syscfg/index_header.md
diff --git a/doxygen/siphon_templates/syscfg/item_format.md b/doxygen/siphon_templates/markdown/syscfg/item_format.md
index 53136115bbc..53136115bbc 100644
--- a/doxygen/siphon_templates/syscfg/item_format.md
+++ b/doxygen/siphon_templates/markdown/syscfg/item_format.md