aboutsummaryrefslogtreecommitdiffstats
path: root/docs/_scripts
diff options
context:
space:
mode:
Diffstat (limited to 'docs/_scripts')
-rw-r--r--docs/_scripts/Makefile212
-rwxr-xr-xdocs/_scripts/filter_api.py49
-rwxr-xr-xdocs/_scripts/filter_c.py87
-rwxr-xr-xdocs/_scripts/filter_h.py56
-rw-r--r--docs/_scripts/includes_renderer.py78
-rwxr-xr-xdocs/_scripts/prepare-for-site.sh18
-rwxr-xr-xdocs/_scripts/publish-docs.sh85
-rwxr-xr-xdocs/_scripts/siphon-generate79
-rwxr-xr-xdocs/_scripts/siphon-process90
-rw-r--r--docs/_scripts/siphon/__init__.py24
-rw-r--r--docs/_scripts/siphon/generate.py306
-rw-r--r--docs/_scripts/siphon/generate_clicmd.py24
-rw-r--r--docs/_scripts/siphon/generate_syscfg.py24
-rw-r--r--docs/_scripts/siphon/parsers.py150
-rw-r--r--docs/_scripts/siphon/process.py407
-rw-r--r--docs/_scripts/siphon/process_clicmd.py73
-rw-r--r--docs/_scripts/siphon/process_syscfg.py31
-rw-r--r--docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md15
-rw-r--r--docs/_scripts/siphon_templates/markdown/clicmd/index_header.md23
-rw-r--r--docs/_scripts/siphon_templates/markdown/clicmd/index_section.md16
-rw-r--r--docs/_scripts/siphon_templates/markdown/clicmd/item_format.md62
-rw-r--r--docs/_scripts/siphon_templates/markdown/clicmd/item_header.md20
-rw-r--r--docs/_scripts/siphon_templates/markdown/default/index_entry.md16
-rw-r--r--docs/_scripts/siphon_templates/markdown/default/index_section.md18
-rw-r--r--docs/_scripts/siphon_templates/markdown/default/item_format.md16
-rw-r--r--docs/_scripts/siphon_templates/markdown/default/item_header.md18
-rw-r--r--docs/_scripts/siphon_templates/markdown/syscfg/index_header.md111
-rw-r--r--docs/_scripts/siphon_templates/markdown/syscfg/item_format.md42
28 files changed, 2150 insertions, 0 deletions
diff --git a/docs/_scripts/Makefile b/docs/_scripts/Makefile
new file mode 100644
index 00000000000..dbd88bf9a20
--- /dev/null
+++ b/docs/_scripts/Makefile
@@ -0,0 +1,212 @@
+# Copyright (c) 2021 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Build the documentation
+#
+
+# Default target
+.PHONY: all
+all: siphon
+
+# These should be passed in by the root Makefile
+WS_ROOT ?= $(CURDIR)/../..
+BR ?= $(WS_ROOT)/build-root
+
+# Tag used in github repository path.
+# Change this when genearting for a release
+VPP_TAG ?= master
+
+REPOSITORY_URL ?= https://github.com/FDio/vpp/blob/$(VPP_TAG)/
+
+# Doxygen configuration and our utility scripts
+SCRIPTS_DIR ?= $(WS_ROOT)/docs/_scripts
+
+# docs root directory
+DOCS_DIR ?= ${BR}/docs/src
+
+FEATURE_LIST_FILE = ${DOCS_DIR}/aboutvpp/featurelist.md
+
+# Siphoned fragements are processed into here
+DOCS_GENERATED_DIR ?= $(DOCS_DIR)/_generated
+
+# Siphoned fragments end up in here
+SIPHON_INPUT_DIR ?= $(DOCS_GENERATED_DIR)/fragments
+
+DYNAMIC_RENDER_DIR ?= ${DOCS_GENERATED_DIR}/includes
+
+# Primary source directories
+SIPHON_SRC ?= $(WS_ROOT)/src
+SIPHON_SRC_DIRECTORIES = \
+ $(shell find $(SIPHON_SRC) -name '*.md' -print | xargs dirname \
+ | sort | uniq) \
+ $(SIPHON_SRC)/vppinfra \
+ $(SIPHON_SRC)/svm \
+ $(SIPHON_SRC)/vlib \
+ $(SIPHON_SRC)/vlibapi \
+ $(SIPHON_SRC)/vlibmemory \
+ $(SIPHON_SRC)/vnet \
+ $(SIPHON_SRC)/vpp \
+ $(SIPHON_SRC)/vpp-api \
+ $(SIPHON_SRC)/examples
+
+# Input directories and files
+SIPHON_INPUT ?= \
+ $(wildcard $(WS_ROOT)/*.md) \
+ $(wildcard $(SCRIPTS_DIR)/*.md) \
+ $(SIPHON_SRC_DIRECTORIES) \
+ $(SIPHON_SRC)/plugins \
+ extras
+
+# Strip leading workspace path from input names
+SIPHON_INPUT := $(subst $(WS_ROOT)/,,$(SIPHON_INPUT))
+
+# Files to exclude, from pre-Doxygen steps, eg because they're
+# selectively compiled.
+# Examples would be to exclude non-DPDK related sources when
+# there's a DPDK equivalent that conflicts.
+# These must be left-anchored paths for the regexp below to work.
+SIPHON_EXCLUDE ?= \
+ $(SIPHON_SRC)/vpp-api/lua
+
+# Generate a regexp for filenames to exclude
+SIPHON_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(SIPHON_EXCLUDE))' | sed -e 's/ /|/g')))
+
+# Include all the normal source directories in the include file path
+SIPHON_INCLUDE_PATH = $(SIPHON_SRC_DIRECTORIES)
+
+# Find API header directories and include them in the header path.
+# This is only useful if VPP and plugins are already built; nothing
+# here depends on those targets. We don't build documentation for these
+# header files, they're just added to the INCLUDE search path for Doxygen.
+_vpp_br = $(shell find "$(BR)" -maxdepth 1 -type d \
+ '(' -name build-vpp_debug-native -o -name build-vpp-native ')' -print \
+ | sed -e 's@^$(WS_ROOT)/*@@' -e 1q)
+ifneq ($(strip $(_vpp_br)),)
+SIPHON_INCLUDE_PATH += \
+ $(_vpp_br)/vlib-api \
+ $(_vpp_br)/vpp
+# Also include any plugin directories that exist
+SIPHON_INCLUDE_PATH += \
+ $(shell find $(WS_ROOT)/$(_vpp_br)/plugins -maxdepth 1 -type d | sed -e 's@^$(WS_ROOT)/*@@')
+endif
+
+# Discover if we have CPP available
+_cpp = $(shell which cpp)
+ifneq ($(strip $(_cpp)),)
+# Add whatever directories CPP normally includes to the header path
+SIPHON_INCLUDE_PATH += $(shell set -e; $(_cpp) -v </dev/null 2>&1 | awk 'f&&/^ /{print $$1} /^\#include/{f=1}')
+endif
+
+# All the siphon types we know about
+SIPHONS ?= clicmd syscfg
+
+SIPHON_FILES = $(addprefix $(SIPHON_INPUT_DIR)/,$(addsuffix .siphon,$(SIPHONS)))
+SIPHON_DOCS = $(addprefix $(DOCS_GENERATED_DIR)/,$(addsuffix .rst,$(SIPHONS)))
+
+BUILT_ON = $(shell date '+%d %B %Y')
+VPP_VERSION = $(shell ${WS_ROOT}/src/scripts/version)
+
+.PHONY: featurelist
+featurelist:
+ @( \
+ cd $(WS_ROOT) && \
+ find . -name FEATURE.yaml | \
+ ./src/scripts/fts.py \
+ --markdown \
+ --repolink $(REPOSITORY_URL) > \
+ $(FEATURE_LIST_FILE) ; \
+ )
+
+
+.PHONY: includes-render
+includes-render:
+ @mkdir -p "$(DYNAMIC_RENDER_DIR)"
+ @python3 $(SCRIPTS_DIR)/includes_renderer.py ${WS_ROOT} ${DYNAMIC_RENDER_DIR}
+
+.PHONY: template-index
+template-index:
+ @sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${DOCS_DIR}/index.rst
+ @sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${DOCS_DIR}/index.rst
+
+.NOTPARALLEL: $(SIPHON_FILES)
+$(SIPHON_FILES): $(SCRIPTS_DIR)/siphon-generate \
+ $(addprefix,$(WSROOT),$(SIPHON_INPUT)) \
+ $(wildcard $(SCRIPTS_DIR)/siphon/*.py)
+ @echo "Validating source tree..."
+ @set -e; for input in $(SIPHON_INPUT); do \
+ if [ ! -e "$(WS_ROOT)/$$input" ]; then \
+ echo "ERROR: Input path '$$input' does not exist." >&2; \
+ exit 1; \
+ fi; \
+ done
+ @rm -rf "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+ @mkdir -p "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)"
+ @touch $(SIPHON_INPUT_DIR)/files
+ @echo "Collating source file list for siphoning..."
+ @for input in $(SIPHON_INPUT); do \
+ cd "$(WS_ROOT)"; \
+ find "$$input" -type f \
+ \( -name '*.[ch]' -or -name '*.dox' \) -print \
+ | grep -v -E '^src/examples/' \
+ | grep -v -E '^$(SIPHON_EXCLUDE_REGEXP)' \
+ >> $(SIPHON_INPUT_DIR)/files; \
+ done
+ @echo "Generating siphons..."
+ @set -e; \
+ cd "$(WS_ROOT)"; \
+ $(SCRIPTS_DIR)/siphon-generate \
+ --output="$(SIPHON_INPUT_DIR)" \
+ "@$(SIPHON_INPUT_DIR)/files"
+
+# Evaluate this to build a siphon doc output target for each desired
+# output type:
+# $(eval $(call siphon-process,file_extension,output_type_name))
+define siphon-process
+$(DOCS_GENERATED_DIR)/%.$(1): $(SIPHON_INPUT_DIR)/%.siphon \
+ $(SCRIPTS_DIR)/siphon-process \
+ $(wildcard $(SCRIPTS_DIR)/siphon/*.py) \
+ $(wildcard $(SCRIPTS_DIR)/siphon_templates/$(2)/*/*.$(1))
+ @echo "Processing siphon for $(2) from $$(notdir $$<)..."
+ @set -e; \
+ cd "$(WS_ROOT)"; \
+ mkdir -p $(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir; \
+ $(SCRIPTS_DIR)/siphon-process \
+ --type=$$(basename $$(notdir $$<)) \
+ --format=$(2) \
+ --repolink=$(REPOSITORY_URL)/ \
+ --outdir=$(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir \
+ --output="$$@" \
+ "$$<"
+endef
+
+# Process the .siphon source fragments and render them into siphon flavored
+# markdown documentation
+.DELETE_ON_ERROR: $(SIPHON_DOCS)
+$(eval $(call siphon-process,rst,markdown))
+
+# This target can be used just to generate the siphoned things
+.PHONY: siphon
+siphon: $(SIPHON_DOCS)
+ @cp $(DOCS_GENERATED_DIR)/clicmd.rst $(DOCS_DIR)/cli-reference/index.rst
+ @cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir $(DOCS_DIR)/cli-reference/clis
+
+.PHONY: generate
+generate: siphon includes-render template-index featurelist
+
+.PHONY: clean
+clean:
+ @rm -rf $(BR)/.siphon.dep
+ @rm -rf $(SCRIPTS_DIR)/siphon/__pycache__
+
diff --git a/docs/_scripts/filter_api.py b/docs/_scripts/filter_api.py
new file mode 100755
index 00000000000..484881439b8
--- /dev/null
+++ b/docs/_scripts/filter_api.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for vpe.api to make it Doxygenish.
+
+import re
+import sys
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+patterns = [
+ # Search for "define" blocks and treat them as structs
+ (re.compile(r"^.*(manual_.[^\s]+\s+)?define\s+(?P<name>[^\s]+)"),
+ r"typedef struct vl_api_\g<name>_t"),
+
+ # For every "brief" statement at the start of a comment block, add an
+ # xref with whatever is on the same line. This gives us an index page
+ # with all the API methods in one place.
+ # XXX Commented out for now; works but duplicates the brief text in the
+ # struct documentation
+ # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)(\*/)$"),
+ # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'), # capture inline comment close
+ # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)$"),
+ # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'),
+
+ # Since structs don't have params, replace @param with @tparam
+ ( re.compile("[\\@]param\\b"), "@tparam"),
+]
+
+with open(sys.argv[1]) as fd:
+ for line in fd:
+ str = line[:-1] # strip \n
+ for p in patterns:
+ str = p[0].sub(p[1], str)
+ sys.stdout.write(str+"\n")
diff --git a/docs/_scripts/filter_c.py b/docs/_scripts/filter_c.py
new file mode 100755
index 00000000000..897f9f6d0b3
--- /dev/null
+++ b/docs/_scripts/filter_c.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .c files to make various preprocessor tricks Doxygenish
+
+import os, sys, re
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+replace_patterns = [
+ # Search for VLIB_CLI_COMMAND, extract its parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_cli_command_t \g<name>"),
+ ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_cli_command_t \g<name>"),
+
+ # Search for VLIB_REGISTER_NODE, extract its parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_node_registration_t \g<name>"),
+ ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_node_registration_t \g<name>"),
+
+ # Search for VLIB_INIT_FUNCTION, extract its parameter and add a docblock for it
+ ( re.compile("(?P<m>VLIB_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"),
+ ( re.compile("(?P<m>VLIB_DECLARE_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"),
+
+ # Search for VLIB_LOOP_ENTER_FUNCTION, extract the parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_MAIN_LOOP_ENTER_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_enter_\g<name>"),
+ ( re.compile("(?P<m>VLIB_MAIN_LOOP_EXIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_exit_\g<name>"),
+
+ # Search for VLIB_CONFIG_FUNCTION, extract the parameters and add a docblock for it
+ ( re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"),
+ ( re.compile("(?P<m>VLIB_EARLY_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"),
+ r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"),
+
+ # Search for "format_thing" and "unformat_thing" when used as a function pointer and add parens
+ ( re.compile("(?P<pre>(^|,)\s*)(?P<name>(un)?format_[a-zA-Z0-9_]+)(?P<post>\s*(,|$))"),
+ r"\g<pre>\g<name>()\g<post>" ),
+
+ # Search for CLIB_PAD_FROM_TO(...); and replace with padding
+ # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
+ ( re.compile("(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),\s*(?P<to>[^)]+)[)]"),
+ r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]" ),
+
+]
+
+
+filename = sys.argv[1]
+cwd = os.getcwd()
+if filename[0:len(cwd)] == cwd:
+ filename = filename[len(cwd):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+with open(filename) as fd:
+ line_num = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ # Look for search/replace patterns
+ for p in replace_patterns:
+ str = p[0].sub(p[1], str)
+
+ sys.stdout.write(str+"\n")
+
+# All done
diff --git a/docs/_scripts/filter_h.py b/docs/_scripts/filter_h.py
new file mode 100755
index 00000000000..0891fa708e1
--- /dev/null
+++ b/docs/_scripts/filter_h.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .c files to make various preprocessor tricks Doxygenish
+
+import os
+import re
+import sys
+
+if len(sys.argv) < 2:
+ sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0]))
+ sys.exit(1)
+
+replace_patterns = [
+ # Search for CLIB_PAD_FROM_TO(...); and replace with padding
+ # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)]
+ (re.compile(r"(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),"
+ r"\s*(?P<to>[^)]+)[)]"),
+ r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]"),
+
+]
+
+
+filename = sys.argv[1]
+cwd = os.getcwd()
+if filename[0:len(cwd)] == cwd:
+ filename = filename[len(cwd):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+with open(filename) as fd:
+ line_num = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ # Look for search/replace patterns
+ for p in replace_patterns:
+ str = p[0].sub(p[1], str)
+
+ sys.stdout.write(str+"\n")
+
+# All done
diff --git a/docs/_scripts/includes_renderer.py b/docs/_scripts/includes_renderer.py
new file mode 100644
index 00000000000..6bd501d83ff
--- /dev/null
+++ b/docs/_scripts/includes_renderer.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020. Vinci Consulting Corp. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import inspect
+import os
+import re
+import sys
+
+
+class ContentRenderer:
+ def __init__(self, ws_root, output_dir):
+ self.ws_root = ws_root
+ self.output_dir = output_dir
+
+ def plugin_dir(self):
+ return os.path.join(self.ws_root, "src/plugins")
+
+ def render(self):
+ raise NotImplementedError
+
+
+class PluginRenderer(ContentRenderer):
+
+ def _render_entry(self, output_file, entry):
+ description = "<no-description-found>"
+ # we use glob because a plugin can (ioam for now)
+ # define the plugin definition in
+ # a further subdirectory.
+ path = os.path.join(self.plugin_dir(), entry.name, '**')
+ for f in glob.iglob(path, recursive=True):
+ if not f.endswith('.c'):
+ continue
+ with open(f, "r", encoding="utf-8") as src:
+ for match in self.regex.finditer(src.read()):
+ description = "%s" % (match.group(1))
+
+ output_file.write(f"* {entry.name} - {description}\n")
+
+ def render(self):
+ pattern = r'VLIB_PLUGIN_REGISTER\s?\(\)\s*=\s*{.*\.description\s?=\s?"([^"]*)".*};' # noqa: 501
+ self.regex = re.compile(pattern, re.MULTILINE | re.DOTALL)
+ fname = os.path.join(self.output_dir, "plugin_list.inc")
+ with open(fname, "w") as output_file:
+ with os.scandir(self.plugin_dir()) as pdir:
+ for entry in sorted(pdir, key=lambda entry: entry.name):
+ if not entry.name.startswith('.') and entry.is_dir():
+ self._render_entry(output_file, entry)
+
+
+renderers = [PluginRenderer]
+
+
+def main():
+ if len(sys.argv) != 3:
+ print("You need to pass WS_ROOT and OUTPUT_DIR")
+ exit(1)
+
+ print("rendering dynamic includes...")
+ for renderer in renderers:
+ renderer(*sys.argv[1:]).render()
+ print("done.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/_scripts/prepare-for-site.sh b/docs/_scripts/prepare-for-site.sh
new file mode 100755
index 00000000000..15888ba6e83
--- /dev/null
+++ b/docs/_scripts/prepare-for-site.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d "docs" ]; then
+ echo "This script is meant to be run from the root directory"
+ exit 1;
+fi
+
+for f in $(find ./docs -type l)
+do
+ target=$(readlink $f)
+ rm $f
+ cp $(dirname $f)/$target $(dirname $f)/$(basename $target)
+ echo "Replaced symlink $f"
+done
+
+echo "Cleaning doc build directory"
+make docs-clean
+
diff --git a/docs/_scripts/publish-docs.sh b/docs/_scripts/publish-docs.sh
new file mode 100755
index 00000000000..026ab2f3907
--- /dev/null
+++ b/docs/_scripts/publish-docs.sh
@@ -0,0 +1,85 @@
+#!/bin/bash -ex
+
+# publish-docs.sh
+#
+# This sccript is used to publish the VPP User documents to
+# the FD.io Site.
+#
+# Arguments:
+#
+# $1: The main site repo user name
+# $2: The release branch name for example 1908, 1904 etc.
+
+# Some basic checks
+if [ ! -d "docs" ]; then
+ echo "This script is meant to be run from the root directory."
+ exit 1;
+fi
+
+if [ "$#" -ne 2 ]; then
+ echo "Please specify the site username and branch."
+ exit 1;
+fi
+
+# Get the workspace root
+WS_ROOT=$PWD
+
+# Get the VPP branch and username
+SITE_USERNAME=$1
+VPP_BRANCH=$2
+
+#Build the docs
+make docs-clean
+make docs-venv
+make docs
+
+# Clone the site repo
+rm -fr site
+rm -fr sphinx_env
+git clone ssh://git@github.com/$SITE_USERNAME/site
+cd site
+git submodule update --init --recursive
+git remote add upstream ssh://git@github.com/FDio/site
+git remote -v
+git fetch upstream
+git merge -m "Publish the Docs" upstream/master
+
+# Get the version
+VERSION=`source $WS_ROOT/src/scripts/version`
+VERSION=${VERSION/"~"/"-"}
+
+# Copy the files to the appropriate directory
+SRC_DIR=../docs/_build/html/.
+if [ "$VPP_BRANCH" == "master" ]
+then
+ TARGET_DIR=./static/docs/vpp/master
+ rm -fr $TARGET_DIR
+else
+ TARGET_DIR=./static/docs/vpp/v$VPP_BRANCH
+ rm -fr $TARGET_DIR
+ mkdir -p $TARGET_DIR
+ VERSION=v$VPP_BRANCH
+ rm ./static/docs/vpp/latest
+ ln -s $VERSION ./static/docs/vpp/latest
+fi
+
+# Create a branch for the commit
+git checkout -b $VERSION
+git branch
+
+# Copy the docs
+cp -r $SRC_DIR $TARGET_DIR
+
+# Create the feature list
+pushd ..
+source ./docs/venv/bin/activate
+find . -name FEATURE.yaml | ./src/scripts/fts.py --markdown > site/content/vppProject/vppfeatures/features.md
+deactivate
+popd
+
+# Push the new docs
+git add "*"
+git commit -s -m "Publish docs from VPP $VERSION"
+git push origin "$VERSION"
+
+exit 0
diff --git a/docs/_scripts/siphon-generate b/docs/_scripts/siphon-generate
new file mode 100755
index 00000000000..9b69c52cf17
--- /dev/null
+++ b/docs/_scripts/siphon-generate
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Looks for preprocessor macros with struct initializers and siphons them
+# off into another file for later parsing; ostensibly to generate
+# documentation from struct initializer data.
+
+import argparse
+import logging
+import os
+
+import siphon
+
+DEFAULT_LOGFILE = None
+DEFAULT_LOGLEVEL = "info"
+DEFAULT_OUTPUT = "build-root/docs/siphons"
+DEFAULT_PREFIX = os.getcwd()
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
+ help="Log file [%s]" % DEFAULT_LOGFILE)
+ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
+ choices=["debug", "info", "warning", "error", "critical"],
+ help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+
+ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
+ help="Output directory for .siphon files [%s]" %
+ DEFAULT_OUTPUT)
+ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
+ help="Prefix to strip from input pathnames [%s]" %
+ DEFAULT_PREFIX)
+ap.add_argument("input", nargs='+', metavar="input_file",
+ help="Input C source files")
+args = ap.parse_args()
+
+logging.basicConfig(filename=args.log_file,
+ level=getattr(logging, args.log_level.upper(), None))
+log = logging.getLogger("siphon_generate")
+
+
+generate = siphon.generate.Generate(output_directory=args.output,
+ input_prefix=args.input_prefix)
+
+# Pre-process file names in case they indicate a file with
+# a list of files
+files = []
+for filename in args.input:
+ if filename.startswith('@'):
+ with open(filename[1:], 'r') as fp:
+ lines = fp.readlines()
+ for line in lines:
+ file = line.strip()
+ if file not in files:
+ files.append(file)
+ lines = None
+ else:
+ if filename not in files:
+ files.append(filename)
+
+# Iterate all the input files we've been given
+for filename in files:
+ generate.parse(filename)
+
+# Write the extracted data
+generate.deliver()
+
+# All done
diff --git a/docs/_scripts/siphon-process b/docs/_scripts/siphon-process
new file mode 100755
index 00000000000..cbee1e90786
--- /dev/null
+++ b/docs/_scripts/siphon-process
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Filter for .siphon files that are generated by other filters.
+# The idea is to siphon off certain initializers so that we can better
+# auto-document the contents of that initializer.
+
+import argparse
+import logging
+import os
+import sys
+
+import siphon
+
+DEFAULT_LOGFILE = None
+DEFAULT_LOGLEVEL = "info"
+DEFAULT_SIPHON = "clicmd"
+DEFAULT_FORMAT = "markdown"
+DEFAULT_OUTPUT = None
+DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates"
+DEFAULT_OUTPUT_DIR = os.path.dirname(__file__) + "/siphon_docs"
+DEFAULT_REPO_LINK = "https://github.com/FDio/vpp/blob/master/"
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--log-file", default=DEFAULT_LOGFILE,
+ help="Log file [%s]" % DEFAULT_LOGFILE)
+ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL,
+ choices=["debug", "info", "warning", "error", "critical"],
+ help="Logging level [%s]" % DEFAULT_LOGLEVEL)
+
+ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
+ choices=siphon.process.siphons.keys(),
+ help="Siphon type to process [%s]" % DEFAULT_SIPHON)
+ap.add_argument("--format", '-f', default=DEFAULT_FORMAT,
+ choices=siphon.process.formats.keys(),
+ help="Output format to generate [%s]" % DEFAULT_FORMAT)
+ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT,
+ help="Output file (uses stdout if not defined) [%s]" %
+ DEFAULT_OUTPUT)
+ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES,
+ help="Path to render templates directory [%s]" %
+ DEFAULT_TEMPLATES)
+ap.add_argument("--outdir", metavar="directory", default=DEFAULT_OUTPUT_DIR,
+ help="Path to output rendered parts [%s]" %
+ DEFAULT_OUTPUT_DIR)
+ap.add_argument("--repolink", metavar="repolink", default=DEFAULT_REPO_LINK,
+ help="Link to public repository [%s]" %
+ DEFAULT_REPO_LINK)
+ap.add_argument("input", nargs='+', metavar="input_file",
+ help="Input .siphon files")
+args = ap.parse_args()
+
+logging.basicConfig(filename=args.log_file,
+ level=getattr(logging, args.log_level.upper(), None))
+log = logging.getLogger("siphon_process")
+
+# Determine where to send the generated output
+if args.output is None:
+ out = sys.stdout
+else:
+ out = open(args.output, "w+")
+
+# Get our processor
+klass = siphon.process.siphons[args.type]
+processor = klass(
+ template_directory=args.templates,
+ format=args.format,
+ outdir=args.outdir,
+ repository_link=args.repolink
+)
+
+# Load the input files
+processor.load_json(args.input)
+
+# Process the data
+processor.process(out=out)
+
+# All done
diff --git a/docs/_scripts/siphon/__init__.py b/docs/_scripts/siphon/__init__.py
new file mode 100644
index 00000000000..f6417314d82
--- /dev/null
+++ b/docs/_scripts/siphon/__init__.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Siphon classes
+
+from . import generate
+from . import generate_clicmd
+from . import generate_syscfg
+
+from . import parsers
+from . import process
+from . import process_clicmd
+from . import process_syscfg
diff --git a/docs/_scripts/siphon/generate.py b/docs/_scripts/siphon/generate.py
new file mode 100644
index 00000000000..2ae5a1b6f1b
--- /dev/null
+++ b/docs/_scripts/siphon/generate.py
@@ -0,0 +1,306 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate .siphon source fragments for later processing
+
+import json
+import logging
+import os
+import re
+
+"""List of (regexp, siphon_name) tuples for matching the start of C
+ initializer blocks in source files. Each siphon class registers
+ themselves on this list."""
+siphon_patterns = []
+
+class Generate(object):
+ """Matches a siphon comment block start"""
+ siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
+
+ """Matches a siphon comment block stop"""
+ siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
+
+ """Siphon block directive delimiter"""
+ siphon_block_delimiter = "%%"
+
+ """Matches a siphon block directive such as
+ '%clicmd:group_label Debug CLI%'"""
+ siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
+ (siphon_block_delimiter, siphon_block_delimiter))
+
+ """Matches the start of an initializer block"""
+ siphon_initializer = re.compile("\s*=")
+
+ """Collated output for each siphon"""
+ output = None
+
+ """Directory prefix to strip from input filenames to keep things tidy."""
+ input_prefix = None
+
+ """List of known siphons"""
+ known_siphons = None
+
+ """Logging handler"""
+ log = None
+
+
+ def __init__(self, output_directory, input_prefix):
+ super(Generate, self).__init__()
+ self.log = logging.getLogger("siphon.generate")
+
+ # Build a list of known siphons
+ self.known_siphons = []
+ for item in siphon_patterns:
+ siphon = item[1]
+ if siphon not in self.known_siphons:
+ self.known_siphons.append(siphon)
+
+ # Setup information for siphons we know about
+ self.output = {}
+ for siphon in self.known_siphons:
+ self.output[siphon] = {
+ "file": "%s/%s.siphon" % (output_directory, siphon),
+ "global": {},
+ "items": [],
+ }
+
+ self.input_prefix = input_prefix
+
+
+ """
+ count open and close braces in str
+ return (0, index) when braces were found and count becomes 0.
+ index indicates the position at which the last closing brace was
+ found.
+ return (-1, -1) if a closing brace is found before any opening one.
+ return (count, -1) if not all opening braces are closed, count is the
+ current depth
+ """
+ def count_braces(self, str, count=0, found=False):
+ for index in range(0, len(str)):
+ if str[index] == '{':
+ count += 1;
+ found = True
+ elif str[index] == '}':
+ if count == 0:
+ # means we never found an open brace
+ return (-1, -1)
+ count -= 1;
+
+ if count == 0 and found:
+ return (count, index)
+
+ return (count, -1)
+
+ def parse(self, filename):
+ # Strip the current directory off the start of the
+ # filename for brevity
+ if filename[0:len(self.input_prefix)] == self.input_prefix:
+ filename = filename[len(self.input_prefix):]
+ if filename[0] == "/":
+ filename = filename[1:]
+
+ # Work out the abbreviated directory name
+ directory = os.path.dirname(filename)
+ if directory[0:2] == "./":
+ directory = directory[2:]
+ elif directory[0:len(self.input_prefix)] == self.input_prefix:
+ directory = directory[len(self.input_prefix):]
+ if directory[0] == "/":
+ directory = directory[1:]
+
+ # Open the file and explore its contents...
+ self.log.info("Siphoning from %s." % filename)
+ directives = {}
+ with open(filename) as fd:
+ siphon = None
+ close_siphon = None
+ siphon_block = ""
+ in_block = False
+ line_num = 0
+ siphon_line = 0
+
+ for line in fd:
+ line_num += 1
+ str = line[:-1] # filter \n
+
+ """See if there is a block directive and if so extract it"""
+ def process_block_directive(str, directives):
+ m = self.siphon_block_directive.search(str)
+ if m is not None:
+ k = m.group(2)
+ v = m.group(3).strip()
+ directives[k] = v
+ # Return only the parts we did not match
+ return str[0:m.start(1)] + str[m.end(4):]
+
+ return str
+
+ def process_block_prefix(str):
+ if str.startswith(" * "):
+ str = str[3:]
+ elif str == " *":
+ str = ""
+ return str
+
+ if not in_block:
+ # See if the line contains the start of a siphon doc block
+ m = self.siphon_block_start.search(str)
+ if m is not None:
+ in_block = True
+ t = m.group(1)
+
+ # Now check if the block closes on the same line
+ m = self.siphon_block_stop.search(t)
+ if m is not None:
+ t = m.group(1)
+ in_block = False
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t
+
+ # Skip to next line
+ continue
+
+ else:
+ # Check to see if we have an end block marker
+ m = self.siphon_block_stop.search(str)
+ if m is not None:
+ in_block = False
+ t = m.group(1)
+ else:
+ t = str
+
+ # Check for directives
+ t = process_block_directive(t, directives)
+
+ # Filter for normal comment prefixes
+ t = process_block_prefix(t)
+
+ # Add what is left
+ siphon_block += t + "\n"
+
+ # Skip to next line
+ continue
+
+
+ if siphon is None:
+ # Look for blocks we need to siphon
+ for p in siphon_patterns:
+ if p[0].match(str):
+ siphon = [ p[1], str + "\n", 0 ]
+ siphon_line = line_num
+
+ # see if we have an initializer
+ m = self.siphon_initializer.search(str)
+ if m is not None:
+ # count the braces on this line
+ (count, index) = \
+ self.count_braces(str[m.start():])
+ siphon[2] = count
+ # TODO - it's possible we have the
+ # initializer all on the first line
+ # we should check for it, but also
+ # account for the possibility that
+ # the open brace is on the next line
+ #if count == 0:
+ # # braces balanced
+ # close_siphon = siphon
+ # siphon = None
+ else:
+ # no initializer: close the siphon right now
+ close_siphon = siphon
+ siphon = None
+ else:
+ # See if we should end the siphon here - do we have
+ # balanced braces?
+ (count, index) = self.count_braces(str,
+ count=siphon[2], found=True)
+ if count == 0:
+ # braces balanced - add the substring and
+ # close the siphon
+ siphon[1] += str[:index+1] + ";\n"
+ close_siphon = siphon
+ siphon = None
+ else:
+ # add the whole string, move on
+ siphon[2] = count
+ siphon[1] += str + "\n"
+
+ if close_siphon is not None:
+ # Write the siphoned contents to the right place
+ siphon_name = close_siphon[0]
+
+ # Copy directives for the file
+ details = {}
+ for key in directives:
+ if ":" in key:
+ (sn, label) = key.split(":")
+ if sn == siphon_name:
+ details[label] = directives[key]
+ else:
+ details[key] = directives[key]
+
+ # Copy details for this block
+ details['file'] = filename
+ details['directory'] = directory
+ details['line_start'] = siphon_line
+ details['line_end'] = line_num
+ details['siphon_block'] = siphon_block.strip()
+ details["block"] = close_siphon[1]
+
+ # Store the item
+ self.output[siphon_name]['items'].append(details)
+
+ # All done
+ close_siphon = None
+ siphon_block = ""
+
+ # Update globals
+ for key in directives.keys():
+ if ':' not in key:
+ continue
+
+ if filename.endswith("/dir.dox"):
+ # very special! use the parent directory name
+ l = directory
+ else:
+ l = filename
+
+ (sn, label) = key.split(":")
+
+ if sn not in self.output:
+ self.output[sn] = {}
+ if 'global' not in self.output[sn]:
+ self.output[sn]['global'] = {}
+ if l not in self.output[sn]['global']:
+ self.output[sn]['global'][l] = {}
+
+ self.output[sn]['global'][l][label] = directives[key]
+
+ def deliver(self):
+ # Write out the data
+ for siphon in self.output.keys():
+ self.log.info("Saving siphon data %s." % siphon)
+ s = self.output[siphon]
+ with open(s['file'], "a") as fp:
+ json.dump(s, fp,
+ separators=(',', ': '), indent=4, sort_keys=True)
+
diff --git a/docs/_scripts/siphon/generate_clicmd.py b/docs/_scripts/siphon/generate_clicmd.py
new file mode 100644
index 00000000000..6d24aaf4926
--- /dev/null
+++ b/docs/_scripts/siphon/generate_clicmd.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from . import generate
+
+# Register our regexp
+generate.siphon_patterns.append((
+ re.compile("(?P<m>VLIB_CLI_COMMAND)\s*"
+ "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"),
+ "clicmd"
+))
diff --git a/docs/_scripts/siphon/generate_syscfg.py b/docs/_scripts/siphon/generate_syscfg.py
new file mode 100644
index 00000000000..52c802e5752
--- /dev/null
+++ b/docs/_scripts/siphon/generate_syscfg.py
@@ -0,0 +1,24 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from . import generate
+
+# Register our regexp
+generate.siphon_patterns.append((
+ re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*"
+ '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'),
+ "syscfg"
+))
diff --git a/docs/_scripts/siphon/parsers.py b/docs/_scripts/siphon/parsers.py
new file mode 100644
index 00000000000..162205de4ca
--- /dev/null
+++ b/docs/_scripts/siphon/parsers.py
@@ -0,0 +1,150 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import html
+import pyparsing as pp
+
+# Some useful primitives
+ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
+intNum = pp.Word(pp.nums)
+hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
+octalNum = pp.Literal("0") + pp.Word("01234567")
+integer = (hexNum | octalNum | intNum) + \
+ pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
+floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
+char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
+arrayIndex = integer | ident
+
+lbracket = pp.Literal("(").suppress()
+rbracket = pp.Literal(")").suppress()
+lbrace = pp.Literal("{").suppress()
+rbrace = pp.Literal("}").suppress()
+comma = pp.Literal(",").suppress()
+equals = pp.Literal("=").suppress()
+dot = pp.Literal(".").suppress()
+semicolon = pp.Literal(";").suppress()
+
+# initializer := { [member = ] (variable | expression | { initializer } ) }
+typeName = ident
+varName = ident
+typeSpec = pp.Optional("unsigned") + \
+ pp.oneOf("int long short float double char u8 i8 void") + \
+ pp.Optional(pp.Word("*"), default="")
+typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
+
+string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
+ escChar='\\', multiline=True)), adjacent=False)
+literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
+var = pp.Combine(pp.Optional(typeCast) + varName +
+ pp.Optional("[" + arrayIndex + "]"))
+
+# This could be more complete, but suffices for our uses
+expr = (literal | var)
+
+"""Parse and render a block of text into a Python dictionary."""
+class Parser(object):
+ """Compiled PyParsing BNF"""
+ _parser = None
+
+ def __init__(self):
+ super(Parser, self).__init__()
+ self._parser = self.BNF()
+
+ def BNF(self):
+ raise NotImplementedError
+
+ def item(self, item):
+ raise NotImplementedError
+
+ def parse(self, input):
+ item = self._parser.parseString(input).asList()
+ return self.item(item)
+
+
+"""Parser for function-like macros - without the closing semi-colon."""
+class ParserFunctionMacro(Parser):
+ def BNF(self):
+ # VLIB_CONFIG_FUNCTION (unix_config, "unix")
+ macroName = ident
+ params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
+ macroParams = lbracket + params + rbracket
+
+ return macroName + macroParams
+
+ def item(self, item):
+ r = {
+ "macro": item[0],
+ "name": item[1][1],
+ "function": item[1][0],
+ }
+
+ return r
+
+
+"""Parser for function-like macros with a closing semi-colon."""
+class ParseFunctionMacroStmt(ParserFunctionMacro):
+ def BNF(self):
+ # VLIB_CONFIG_FUNCTION (unix_config, "unix");
+ function_macro = super(ParseFunctionMacroStmt, self).BNF()
+ mi = function_macro + semicolon
+ mi.ignore(pp.cppStyleComment)
+
+ return mi
+
+
+"""
+Parser for our struct initializers which are composed from a
+function-like macro, equals sign, and then a normal C struct initializer
+block.
+"""
+class MacroInitializer(ParserFunctionMacro):
+ def BNF(self):
+ # VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
+ # .path = "show sr tunnel",
+ # .short_help = "show sr tunnel [name <sr-tunnel-name>]",
+ # .function = show_sr_tunnel_fn,
+ # };
+ cs = pp.Forward()
+
+
+ member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"),
+ adjacent=False)
+ value = (expr | cs)
+
+ entry = pp.Group(pp.Optional(member + equals, default="") + value)
+ entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
+ (pp.ZeroOrMore(entry + comma))
+
+ cs << (lbrace + entries + rbrace)
+
+ macroName = ident
+ params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
+ macroParams = lbracket + params + rbracket
+
+ function_macro = super(MacroInitializer, self).BNF()
+ mi = function_macro + equals + pp.Group(cs) + semicolon
+ mi.ignore(pp.cppStyleComment)
+
+ return mi
+
+ def item(self, item):
+ r = {
+ "macro": item[0],
+ "name": item[1][0],
+ "params": item[2],
+ "value": {},
+ }
+
+ for param in item[2]:
+ r["value"][param[0]] = html.escape(param[1])
+
+ return r
diff --git a/docs/_scripts/siphon/process.py b/docs/_scripts/siphon/process.py
new file mode 100644
index 00000000000..e3a70152487
--- /dev/null
+++ b/docs/_scripts/siphon/process.py
@@ -0,0 +1,407 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generation template class
+
+import html.parser
+import json
+import logging
+import os
+import sys
+import re
+
+import jinja2
+
+# Classes register themselves in this dictionary
+"""Mapping of known processors to their classes"""
+siphons = {}
+
+"""Mapping of known output formats to their classes"""
+formats = {}
+
+
+class Siphon(object):
+ """Generate rendered output for siphoned data."""
+
+ # Set by subclasses
+ """Our siphon name"""
+ name = None
+
+ # Set by subclasses
+ """Name of an identifier used by this siphon"""
+ identifier = None
+
+ # Set by subclasses
+ """The pyparsing object to use to parse with"""
+ _parser = None
+
+ """The input data"""
+ _cmds = None
+
+ """Group key to (directory,file) mapping"""
+ _group = None
+
+ """Logging handler"""
+ log = None
+
+ """Directory to look for siphon rendering templates"""
+ template_directory = None
+
+ """Directory to output parts in"""
+ outdir = None
+
+ """Template environment, if we're using templates"""
+ _tplenv = None
+
+ def __init__(self, template_directory, format, outdir, repository_link):
+ super(Siphon, self).__init__()
+ self.log = logging.getLogger("siphon.process.%s" % self.name)
+
+ # Get our output format details
+ fmt_klass = formats[format]
+ fmt = fmt_klass()
+ self._format = fmt
+
+ # Sort out the template search path
+ def _tpldir(name):
+ return os.sep.join((template_directory, fmt.name, name))
+
+ self.template_directory = template_directory
+ searchpath = [
+ _tpldir(self.name),
+ _tpldir("default"),
+ ]
+ self.outdir = outdir
+ loader = jinja2.FileSystemLoader(searchpath=searchpath)
+ self._tplenv = jinja2.Environment(
+ loader=loader,
+ trim_blocks=True,
+ autoescape=False,
+ keep_trailing_newline=True)
+
+ # Convenience, get a reference to the internal escape and
+ # unescape methods in html.parser. These then become
+ # available to templates to use, if needed.
+ self._h = html.parser.HTMLParser()
+ self.escape = html.escape
+ self.unescape = html.unescape
+
+ # TODO: customize release
+ self.repository_link = repository_link
+
+ # Output renderers
+
+ """Returns an object to be used as the sorting key in the item index."""
+ def index_sort_key(self, group):
+ return group
+
+ """Returns a string to use as the header at the top of the item index."""
+ def index_header(self):
+ return self.template("index_header")
+
+ """Returns the string fragment to use for each section in the item
+ index."""
+ def index_section(self, group):
+ return self.template("index_section", group=group)
+
+ """Returns the string fragment to use for each entry in the item index."""
+ def index_entry(self, meta, item):
+ return self.template("index_entry", meta=meta, item=item)
+
+ """Returns an object, typically a string, to be used as the sorting key
+ for items within a section."""
+ def item_sort_key(self, item):
+ return item['name']
+
+ """Returns a key for grouping items together."""
+ def group_key(self, directory, file, macro, name):
+ _global = self._cmds['_global']
+
+ if file in _global and 'group_label' in _global[file]:
+ self._group[file] = (directory, file)
+ return file
+
+ self._group[directory] = (directory, None)
+ return directory
+
+ """Returns a key for identifying items within a grouping."""
+ def item_key(self, directory, file, macro, name):
+ return name
+
+ """Returns a string to use as the header when rendering the item."""
+ def item_header(self, group):
+ return self.template("item_header", group=group)
+
+ """Returns a string to use as the body when rendering the item."""
+ def item_format(self, meta, item):
+ return self.template("item_format", meta=meta, item=item)
+
+ """Returns a string to use as the label for the page reference."""
+ def page_label(self, group):
+ return "_".join((
+ self.name,
+ self.sanitize_label(group)
+ ))
+
+ """Returns a title to use for a page."""
+ def page_title(self, group):
+ _global = self._cmds['_global']
+ (directory, file) = self._group[group]
+
+ if file and file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ return directory
+
+ """Returns a string to use as the label for the section reference."""
+ def item_label(self, group, item):
+ return "__".join((
+ self.name,
+ item
+ ))
+
+ """Label sanitizer; for creating Doxygen references"""
+ def sanitize_label(self, value):
+ return value.replace(" ", "_") \
+ .replace("/", "_") \
+ .replace(".", "_")
+
+ """Template processor"""
+ def template(self, name, **kwargs):
+ tpl = self._tplenv.get_template(name + self._format.extension)
+ return tpl.render(
+ this=self,
+ **kwargs)
+
+ # Processing methods
+
+ """Parse the input file into a more usable dictionary structure."""
+ def load_json(self, files):
+ self._cmds = {}
+ self._group = {}
+
+ line_num = 0
+ line_start = 0
+ for filename in files:
+ filename = os.path.relpath(filename)
+ self.log.info("Parsing items in file \"%s\"." % filename)
+ data = None
+ with open(filename, "r") as fd:
+ data = json.load(fd)
+
+ self._cmds['_global'] = data['global']
+
+ # iterate the items loaded and regroup it
+ for item in data["items"]:
+ try:
+ o = self._parser.parse(item['block'])
+ except Exception:
+ self.log.error("Exception parsing item: %s\n%s"
+ % (json.dumps(item, separators=(',', ': '),
+ indent=4),
+ item['block']))
+ raise
+
+ # Augment the item with metadata
+ o["meta"] = {}
+ for key in item:
+ if key == 'block':
+ continue
+ o['meta'][key] = item[key]
+
+ # Load some interesting fields
+ directory = item['directory']
+ file = item['file']
+ macro = o["macro"]
+ name = o["name"]
+
+ # Generate keys to group items by
+ group_key = self.group_key(directory, file, macro, name)
+ item_key = self.item_key(directory, file, macro, name)
+
+ if group_key not in self._cmds:
+ self._cmds[group_key] = {}
+
+ self._cmds[group_key][item_key] = o
+
+ """Iterate over the input data, calling render methods to generate the
+ output."""
+ def process(self, out=None):
+
+ if out is None:
+ out = sys.stdout
+
+ # Accumulated body contents
+ contents = ""
+
+ # Write the header for this siphon type
+ out.write(self.index_header())
+
+ # Sort key helper for the index
+ def group_sort_key(group):
+ return self.index_sort_key(group)
+
+ # Iterate the dictionary and process it
+ for group in sorted(self._cmds.keys(), key=group_sort_key):
+ if group.startswith('_'):
+ continue
+
+ self.log.info("Processing items in group \"%s\" (%s)." %
+ (group, group_sort_key(group)))
+
+ # Generate the section index entry (write it now)
+ out.write(self.index_section(group))
+
+ # Generate the item header (save for later)
+ contents += self.item_header(group)
+
+ def item_sort_key(key):
+ return self.item_sort_key(self._cmds[group][key])
+
+ for key in sorted(self._cmds[group].keys(), key=item_sort_key):
+ self.log.debug("--- Processing key \"%s\" (%s)." %
+ (key, item_sort_key(key)))
+
+ o = self._cmds[group][key]
+ meta = {
+ "directory": o['meta']['directory'],
+ "file": o['meta']['file'],
+ "macro": o['macro'],
+ "name": o['name'],
+ "key": key,
+ "label": self.item_label(group, key),
+ }
+
+ # Generate the index entry for the item (write it now)
+ out.write(self.index_entry(meta, o))
+
+ # Generate the item itself (save for later)
+ contents += self.item_format(meta, o)
+
+ page_name = self.separate_page_names(group)
+ if page_name != "":
+ path = os.path.join(self.outdir, page_name)
+ with open(path, "w+") as page:
+ page.write(contents)
+ contents = ""
+
+ # Deliver the accumulated body output
+ out.write(contents)
+
+ def do_cliexstart(self, matchobj):
+ title = matchobj.group(1)
+ title = ' '.join(title.splitlines())
+ content = matchobj.group(2)
+ content = re.sub(r"\n", r"\n ", content)
+ return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content)
+
+ def do_clistart(self, matchobj):
+ content = matchobj.group(1)
+ content = re.sub(r"\n", r"\n ", content)
+ return "\n\n.. code-block:: console\n\n %s\n\n" % content
+
+ def do_cliexcmd(self, matchobj):
+ content = matchobj.group(1)
+ content = ' '.join(content.splitlines())
+ return "\n\n.. code-block:: console\n\n %s\n\n" % content
+
+ def process_list(self, matchobj):
+ content = matchobj.group(1)
+ content = self.reindent(content, 2)
+ return "@@@@%s\nBBBB" % content
+
+ def process_special(self, s):
+ # ----------- markers to remove
+ s = re.sub(r"@cliexpar\s*", r"", s)
+ s = re.sub(r"@parblock\s*", r"", s)
+ s = re.sub(r"@endparblock\s*", r"", s)
+ s = re.sub(r"<br>", "", s)
+ # ----------- emphasis
+ # <b><em>
+ s = re.sub(r"<b><em>\s*", "``", s)
+ s = re.sub(r"\s*</b></em>", "``", s)
+ s = re.sub(r"\s*</em></b>", "``", s)
+ # <b>
+ s = re.sub(r"<b>\s*", "**", s)
+ s = re.sub(r"\s*</b>", "**", s)
+ # <code>
+ s = re.sub(r"<code>\s*", "``", s)
+ s = re.sub(r"\s*</code>", "``", s)
+ # <em>
+ s = re.sub(r"'?<em>\s*", r"``", s)
+ s = re.sub(r"\s*</em>'?", r"``", s)
+ # @c <something>
+ s = re.sub(r"@c\s(\S+)", r"``\1``", s)
+ # ----------- todos
+ s = re.sub(r"@todo[^\n]*", "", s)
+ s = re.sub(r"@TODO[^\n]*", "", s)
+ # ----------- code blocks
+ s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL)
+ s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL)
+ s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL)
+ # ----------- lists
+ s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE)
+ s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL)
+ s = re.sub(r"BBBB@@@@", r"-", s)
+ s = re.sub(r"@@@@", r"-", s)
+ s = re.sub(r"BBBB", r"\n\n", s)
+ # ----------- Cleanup remains
+ s = re.sub(r"@cliexend\s*", r"", s)
+ return s
+
+ def separate_page_names(self, group):
+ return ""
+
+ # This push the given textblock <indent> spaces right
+ def reindent(self, s, indent):
+ ind = " " * indent
+ s = re.sub(r"\n", "\n" + ind, s)
+ return s
+
+ # This aligns the given textblock left (no indent)
+ def noindent(self, s):
+ s = re.sub(r"\n[ \f\v\t]*", "\n", s)
+ return s
+
+class Format(object):
+ """Output format class"""
+
+ """Name of this output format"""
+ name = None
+
+ """Expected file extension of templates that build this format"""
+ extension = None
+
+
+class FormatMarkdown(Format):
+ """Markdown output format"""
+ name = "markdown"
+ extension = ".md"
+
+
+# Register 'markdown'
+formats["markdown"] = FormatMarkdown
+
+
+class FormatItemlist(Format):
+ """Itemlist output format"""
+ name = "itemlist"
+ extension = ".itemlist"
+
+
+# Register 'itemlist'
+formats["itemlist"] = FormatItemlist
diff --git a/docs/_scripts/siphon/process_clicmd.py b/docs/_scripts/siphon/process_clicmd.py
new file mode 100644
index 00000000000..bf270518ad1
--- /dev/null
+++ b/docs/_scripts/siphon/process_clicmd.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate clicmd formatted output
+
+from . import process, parsers
+import os
+
+class SiphonCLICMD(process.Siphon):
+
+ name = "clicmd"
+ identifier = "VLIB_CLI_COMMAND"
+
+ def __init__(self, *args, **kwargs):
+ super(SiphonCLICMD, self).__init__(*args, **kwargs)
+ self._parser = parsers.MacroInitializer()
+
+ # Output renderers
+
+ def separate_page_names(self, group):
+ return self.page_label(group) + ".rst"
+
+ def index_sort_key(self, group):
+ _global = self._cmds['_global']
+ if group not in self._group:
+ return group
+ (directory, file) = self._group[group]
+
+ if file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ return group
+
+ def item_sort_key(self, item):
+ return item['value']['path']
+
+ def item_label(self, group, item):
+ return "_".join((
+ self.name,
+ self.sanitize_label(self._cmds[group][item]['value']['path'])
+ ))
+
+ def page_title(self, group):
+ _global = self._cmds['_global']
+ (directory, file) = self._group[group]
+
+ if file and file in _global and 'group_label' in _global[file]:
+ return _global[file]['group_label']
+
+ if directory in _global and 'group_label' in _global[directory]:
+ return _global[directory]['group_label']
+
+ file_ext = os.path.basename(directory)
+ fname, ext = os.path.splitext(file_ext)
+ return "%s cli reference" % fname.capitalize()
+
+
+# Register our processor
+process.siphons["clicmd"] = SiphonCLICMD
diff --git a/docs/_scripts/siphon/process_syscfg.py b/docs/_scripts/siphon/process_syscfg.py
new file mode 100644
index 00000000000..bccde2c153d
--- /dev/null
+++ b/docs/_scripts/siphon/process_syscfg.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate syscfg formatted output
+
+from . import process, parsers
+
+
+class SiphonSYSCFG(process.Siphon):
+
+ name = "syscfg"
+ identifier = "VLIB_CONFIG_FUNCTION"
+
+ def __init__(self, *args, **kwargs):
+ super(SiphonSYSCFG, self).__init__(*args, **kwargs)
+ self._parser = parsers.ParseFunctionMacroStmt()
+
+
+# Register our processor
+process.siphons["syscfg"] = SiphonSYSCFG
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md
new file mode 100644
index 00000000000..de517f160c1
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md
@@ -0,0 +1,15 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#} \ No newline at end of file
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md
new file mode 100644
index 00000000000..86e5bb6c3e5
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md
@@ -0,0 +1,23 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+.. _cmdreference:
+
+Reference
+=========
+
+.. toctree::
+ :maxdepth: 2
+
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md
new file mode 100644
index 00000000000..0da934ac938
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+ clis/{{ this.page_label(group) }}
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md
new file mode 100644
index 00000000000..f1caa961f56
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md
@@ -0,0 +1,62 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{% set v = item['value'] %}
+{% if 'short_help' in v %}
+{% set str = v['short_help'] %}
+{% set str = this.unescape(str) %}
+
+{# Summary/usage #}
+{{ item['value']['path'] }}
+-------------------------------------------------------------------------
+
+.. code-block:: console
+
+ {{ this.reindent(str, 4) }}
+
+{% endif %}
+{% if 'long_help' in v %}
+{% set long_help = v['long_help'] %}
+{% set long_help = this.unescape(long_help) %}
+{# This is seldom used and will likely be deprecated #}
+{# Long help #}
+.. code-block:: console
+
+ {{ this.reindent(long_help, 4) }}
+
+{% endif %}
+{% if 'siphon_block' in item['meta'] %}
+{% set sb = item["meta"]["siphon_block"] %}
+{% set sb = this.process_special(sb) %}
+{% if sb %}
+{# Extracted from the code in /*? ... ?*/ blocks #}
+
+{# Description #}
+
+{{ sb }}
+{% endif %}
+{% endif %}
+{% if 'name' in meta or 'function' in v %}
+{# Gives some developer-useful linking #}
+
+{% if "name" in meta %}
+Declaration: ``{{ meta['name'] }}`` `{{ meta["file"] }} line {{ item["meta"]["line_start"] }} <{{ this.repository_link }}{{ meta["file"] }}#L{{ item["meta"]["line_start"] }}>`_
+{% endif %}
+{% if "function" in v %}
+
+Implementation: ``{{ v["function"] }}``
+{% endif %}
+{% endif %}
+
diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md
new file mode 100644
index 00000000000..20e4803ec2e
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md
@@ -0,0 +1,20 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+.. _{{ this.page_label(group) }}:
+
+===============================================================
+{{ this.page_title(group) }}
+===============================================================
diff --git a/docs/_scripts/siphon_templates/markdown/default/index_entry.md b/docs/_scripts/siphon_templates/markdown/default/index_entry.md
new file mode 100644
index 00000000000..479dcdb2cc3
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/index_entry.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ "* [%s](@ref %s)" % (item["name"], meta["label"]) }}
diff --git a/docs/_scripts/siphon_templates/markdown/default/index_section.md b/docs/_scripts/siphon_templates/markdown/default/index_section.md
new file mode 100644
index 00000000000..3c9d2b474c5
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/index_section.md
@@ -0,0 +1,18 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+@subpage {{ this.page_label(group) }}
+
diff --git a/docs/_scripts/siphon_templates/markdown/default/item_format.md b/docs/_scripts/siphon_templates/markdown/default/item_format.md
new file mode 100644
index 00000000000..ed1b1bf7eeb
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/item_format.md
@@ -0,0 +1,16 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{{ raise NotImplementedError }}
diff --git a/docs/_scripts/siphon_templates/markdown/default/item_header.md b/docs/_scripts/siphon_templates/markdown/default/item_header.md
new file mode 100644
index 00000000000..0c21e51fa5d
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/default/item_header.md
@@ -0,0 +1,18 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{{ "@page %s %s" % (this.page_label(group), this.page_title(group)) }}
+
diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md
new file mode 100644
index 00000000000..f5a9816b99c
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md
@@ -0,0 +1,111 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+# Startup Configuration {{'{#'}}syscfg}
+
+The VPP network stack comes with several configuration options that can be
+provided either on the command line or in a configuration file.
+
+Specific applications built on the stack have been known to require a dozen
+arguments, depending on requirements. This section describes commonly-used
+options and parameters.
+
+You can find command-line argument parsers in the source code by searching for
+instances of the `VLIB_CONFIG_FUNCTION` macro. The invocation
+`VLIB_CONFIG_FUNCTION(foo_config, "foo")` will cause the function
+`foo_config` to receive all the options and values supplied in a parameter
+block named "`foo`", for example: `foo { arg1 arg2 arg3 ... }`.
+
+@todo Tell the nice people where this document lives so that the might
+help improve it!
+
+## Command-line arguments
+
+Parameters are grouped by a section name. When providing more than one
+parameter to a section all parameters for that section must be wrapped in
+curly braces.
+
+```
+/usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 }
+```
+
+Which will produce output similar to this:
+
+ <startup diagnostic messages>
+ _______ _ _ _____ ___
+ __/ __/ _ \ (_)__ | | / / _ \/ _ \
+ _/ _// // / / / _ \ | |/ / ___/ ___/
+ /_/ /____(_)_/\___/ |___/_/ /_/
+
+ vpp# <start-typing>
+
+When providing only one such parameter the braces are optional. For example,
+the following command argument, `unix interactive` does not have braces:
+
+```
+/usr/bin/vpp unix interactive
+```
+
+The command line can be presented as a single string or as several; anything
+given on the command line is concatenated with spaces into a single string
+before parsing.
+
+VPP applications must be able to locate their own executable images. The
+simplest way to ensure this will work is to invoke a VPP application by giving
+its absolute path; for example: `/usr/bin/vpp <options>`. At startup, VPP
+applications parse through their own ELF-sections (primarily) to make lists
+of init, configuration, and exit handlers.
+
+When developing with VPP, in _gdb_ it's often sufficient to start an application
+like this at the `(gdb)` prompt:
+
+```
+run unix interactive
+```
+
+## Configuration file
+
+It is also possible to supply parameters in a startup configuration file the
+path of which is provided to the VPP application on its command line.
+
+The format of the configuration file is a simple text file with the same
+content as the command line but with the benefit of being able to use newlines
+to make the content easier to read. For example:
+
+```
+unix {
+ nodaemon
+ log /var/log/vpp/vpp.log
+ full-coredump
+ cli-listen localhost:5002
+}
+api-trace {
+ on
+}
+dpdk {
+ dev 0000:03:00.0
+}
+```
+
+VPP is then instructed to load this file with the `-c` option:
+
+```
+/usr/bin/vpp -c /etc/vpp/startup.conf
+```
+
+## Index of startup command sections
+
+[TOC]
+
diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md
new file mode 100644
index 00000000000..53136115bbc
--- /dev/null
+++ b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md
@@ -0,0 +1,42 @@
+{#
+# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{% set v = item['value'] %}
+{{ "@section %s %s" % (meta['label'], item['name']) }}
+{% if 'siphon_block' in item['meta'] %}
+{% set sb = item["meta"]["siphon_block"] %}
+{% if sb %}
+{# Extracted from the code in /*? ... ?*/ blocks #}
+
+### Description
+
+{{ sb }}
+{% endif %}
+{% endif %}
+{% if "name" in meta or "function" in item %}
+{# Gives some developer-useful linking #}
+
+### Declaration and implementation
+{% if "name" in meta %}
+
+{{ "Declaration: @ref %s (@ref %s line %d)" %
+ (meta['name'], meta["file"], item["meta"]["line_start"]) }}
+{% endif %}
+{% if "function" in item %}
+
+{{ "Implementation: @ref %s." % item["function"] }}
+{% endif %}
+{% endif %}
+