diff options
| author |   Nathan Skrzypczak <nathan.skrzypczak@gmail.com> | 2021-08-19 11:38:06 +0200 |
|---|---|---|
| committer |   Dave Wallace <dwallacelf@gmail.com> | 2021-10-13 23:22:32 +0000 |
| commit | 9ad39c026c8a3c945a7003c4aa4f5cb1d4c80160 (patch) | |
| tree | 3cca19635417e28ae381d67ae31c75df2925032d /docs/_scripts | |
| parent | f47122e07e1ecd0151902a3cabe46c60a99bee8e (diff) | |
docs: better docs, mv doxygen to sphinx
This patch refactors the VPP sphinx docs
in order to make it easier to consume
for external readers as well as VPP developers.
It also makes sphinx the single source
of documentation, which simplifies maintenance
and operation.
Most important updates are:
- reformat the existing documentation as rst
- split RELEASE.md and move it into separate rst files
- remove section 'events'
- remove section 'archive'
- remove section 'related projects'
- remove section 'feature by release'
- remove section 'Various links'
- make (Configuration reference, CLI docs,
developer docs) top level items in the list
- move 'Use Cases' as part of 'About VPP'
- move 'Troubleshooting' as part of 'Getting Started'
- move test framework docs into 'Developer Documentation'
- add a 'Contributing' section for gerrit,
docs and other contributer related infos
- deprecate doxygen and test-docs targets
- redirect the "make doxygen" target to "make docs"
Type: refactor
Change-Id: I552a5645d5b7964d547f99b1336e2ac24e7c209f
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Diffstat (limited to 'docs/_scripts')
28 files changed, 2150 insertions, 0 deletions
diff --git a/docs/_scripts/Makefile b/docs/_scripts/Makefile new file mode 100644 index 00000000000..dbd88bf9a20 --- /dev/null +++ b/docs/_scripts/Makefile @@ -0,0 +1,212 @@ +# Copyright (c) 2021 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Build the documentation +# + +# Default target +.PHONY: all +all: siphon + +# These should be passed in by the root Makefile +WS_ROOT ?= $(CURDIR)/../.. +BR ?= $(WS_ROOT)/build-root + +# Tag used in github repository path. +# Change this when genearting for a release +VPP_TAG ?= master + +REPOSITORY_URL ?= https://github.com/FDio/vpp/blob/$(VPP_TAG)/ + +# Doxygen configuration and our utility scripts +SCRIPTS_DIR ?= $(WS_ROOT)/docs/_scripts + +# docs root directory +DOCS_DIR ?= ${BR}/docs/src + +FEATURE_LIST_FILE = ${DOCS_DIR}/aboutvpp/featurelist.md + +# Siphoned fragements are processed into here +DOCS_GENERATED_DIR ?= $(DOCS_DIR)/_generated + +# Siphoned fragments end up in here +SIPHON_INPUT_DIR ?= $(DOCS_GENERATED_DIR)/fragments + +DYNAMIC_RENDER_DIR ?= ${DOCS_GENERATED_DIR}/includes + +# Primary source directories +SIPHON_SRC ?= $(WS_ROOT)/src +SIPHON_SRC_DIRECTORIES = \ + $(shell find $(SIPHON_SRC) -name '*.md' -print | xargs dirname \ + | sort | uniq) \ + $(SIPHON_SRC)/vppinfra \ + $(SIPHON_SRC)/svm \ + $(SIPHON_SRC)/vlib \ + $(SIPHON_SRC)/vlibapi \ + $(SIPHON_SRC)/vlibmemory \ + $(SIPHON_SRC)/vnet \ + $(SIPHON_SRC)/vpp \ + $(SIPHON_SRC)/vpp-api \ + $(SIPHON_SRC)/examples + +# Input directories and files +SIPHON_INPUT ?= \ + $(wildcard $(WS_ROOT)/*.md) \ + $(wildcard $(SCRIPTS_DIR)/*.md) \ + $(SIPHON_SRC_DIRECTORIES) \ + $(SIPHON_SRC)/plugins \ + extras + +# Strip leading workspace path from input names +SIPHON_INPUT := $(subst $(WS_ROOT)/,,$(SIPHON_INPUT)) + +# Files to exclude, from pre-Doxygen steps, eg because they're +# selectively compiled. +# Examples would be to exclude non-DPDK related sources when +# there's a DPDK equivalent that conflicts. +# These must be left-anchored paths for the regexp below to work. +SIPHON_EXCLUDE ?= \ + $(SIPHON_SRC)/vpp-api/lua + +# Generate a regexp for filenames to exclude +SIPHON_EXCLUDE_REGEXP = ($(subst .,\.,$(shell echo '$(strip $(SIPHON_EXCLUDE))' | sed -e 's/ /|/g'))) + +# Include all the normal source directories in the include file path +SIPHON_INCLUDE_PATH = $(SIPHON_SRC_DIRECTORIES) + +# Find API header directories and include them in the header path. +# This is only useful if VPP and plugins are already built; nothing +# here depends on those targets. We don't build documentation for these +# header files, they're just added to the INCLUDE search path for Doxygen. +_vpp_br = $(shell find "$(BR)" -maxdepth 1 -type d \ + '(' -name build-vpp_debug-native -o -name build-vpp-native ')' -print \ + | sed -e 's@^$(WS_ROOT)/*@@' -e 1q) +ifneq ($(strip $(_vpp_br)),) +SIPHON_INCLUDE_PATH += \ + $(_vpp_br)/vlib-api \ + $(_vpp_br)/vpp +# Also include any plugin directories that exist +SIPHON_INCLUDE_PATH += \ + $(shell find $(WS_ROOT)/$(_vpp_br)/plugins -maxdepth 1 -type d | sed -e 's@^$(WS_ROOT)/*@@') +endif + +# Discover if we have CPP available +_cpp = $(shell which cpp) +ifneq ($(strip $(_cpp)),) +# Add whatever directories CPP normally includes to the header path +SIPHON_INCLUDE_PATH += $(shell set -e; $(_cpp) -v </dev/null 2>&1 | awk 'f&&/^ /{print $$1} /^\#include/{f=1}') +endif + +# All the siphon types we know about +SIPHONS ?= clicmd syscfg + +SIPHON_FILES = $(addprefix $(SIPHON_INPUT_DIR)/,$(addsuffix .siphon,$(SIPHONS))) +SIPHON_DOCS = $(addprefix $(DOCS_GENERATED_DIR)/,$(addsuffix .rst,$(SIPHONS))) + +BUILT_ON = $(shell date '+%d %B %Y') +VPP_VERSION = $(shell ${WS_ROOT}/src/scripts/version) + +.PHONY: featurelist +featurelist: + @( \ + cd $(WS_ROOT) && \ + find . -name FEATURE.yaml | \ + ./src/scripts/fts.py \ + --markdown \ + --repolink $(REPOSITORY_URL) > \ + $(FEATURE_LIST_FILE) ; \ + ) + + +.PHONY: includes-render +includes-render: + @mkdir -p "$(DYNAMIC_RENDER_DIR)" + @python3 $(SCRIPTS_DIR)/includes_renderer.py ${WS_ROOT} ${DYNAMIC_RENDER_DIR} + +.PHONY: template-index +template-index: + @sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${DOCS_DIR}/index.rst + @sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${DOCS_DIR}/index.rst + +.NOTPARALLEL: $(SIPHON_FILES) +$(SIPHON_FILES): $(SCRIPTS_DIR)/siphon-generate \ + $(addprefix,$(WSROOT),$(SIPHON_INPUT)) \ + $(wildcard $(SCRIPTS_DIR)/siphon/*.py) + @echo "Validating source tree..." + @set -e; for input in $(SIPHON_INPUT); do \ + if [ ! -e "$(WS_ROOT)/$$input" ]; then \ + echo "ERROR: Input path '$$input' does not exist." >&2; \ + exit 1; \ + fi; \ + done + @rm -rf "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)" + @mkdir -p "$(SIPHON_INPUT_DIR)" "$(DOCS_GENERATED_DIR)" + @touch $(SIPHON_INPUT_DIR)/files + @echo "Collating source file list for siphoning..." + @for input in $(SIPHON_INPUT); do \ + cd "$(WS_ROOT)"; \ + find "$$input" -type f \ + \( -name '*.[ch]' -or -name '*.dox' \) -print \ + | grep -v -E '^src/examples/' \ + | grep -v -E '^$(SIPHON_EXCLUDE_REGEXP)' \ + >> $(SIPHON_INPUT_DIR)/files; \ + done + @echo "Generating siphons..." + @set -e; \ + cd "$(WS_ROOT)"; \ + $(SCRIPTS_DIR)/siphon-generate \ + --output="$(SIPHON_INPUT_DIR)" \ + "@$(SIPHON_INPUT_DIR)/files" + +# Evaluate this to build a siphon doc output target for each desired +# output type: +# $(eval $(call siphon-process,file_extension,output_type_name)) +define siphon-process +$(DOCS_GENERATED_DIR)/%.$(1): $(SIPHON_INPUT_DIR)/%.siphon \ + $(SCRIPTS_DIR)/siphon-process \ + $(wildcard $(SCRIPTS_DIR)/siphon/*.py) \ + $(wildcard $(SCRIPTS_DIR)/siphon_templates/$(2)/*/*.$(1)) + @echo "Processing siphon for $(2) from $$(notdir $$<)..." + @set -e; \ + cd "$(WS_ROOT)"; \ + mkdir -p $(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir; \ + $(SCRIPTS_DIR)/siphon-process \ + --type=$$(basename $$(notdir $$<)) \ + --format=$(2) \ + --repolink=$(REPOSITORY_URL)/ \ + --outdir=$(DOCS_GENERATED_DIR)/$$(basename $$(notdir $$<)).$(1).dir \ + --output="$$@" \ + "$$<" +endef + +# Process the .siphon source fragments and render them into siphon flavored +# markdown documentation +.DELETE_ON_ERROR: $(SIPHON_DOCS) +$(eval $(call siphon-process,rst,markdown)) + +# This target can be used just to generate the siphoned things +.PHONY: siphon +siphon: $(SIPHON_DOCS) + @cp $(DOCS_GENERATED_DIR)/clicmd.rst $(DOCS_DIR)/cli-reference/index.rst + @cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir $(DOCS_DIR)/cli-reference/clis + +.PHONY: generate +generate: siphon includes-render template-index featurelist + +.PHONY: clean +clean: + @rm -rf $(BR)/.siphon.dep + @rm -rf $(SCRIPTS_DIR)/siphon/__pycache__ + diff --git a/docs/_scripts/filter_api.py b/docs/_scripts/filter_api.py new file mode 100755 index 00000000000..484881439b8 --- /dev/null +++ b/docs/_scripts/filter_api.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Filter for vpe.api to make it Doxygenish. + +import re +import sys + +if len(sys.argv) < 2: + sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0])) + sys.exit(1) + +patterns = [ + # Search for "define" blocks and treat them as structs + (re.compile(r"^.*(manual_.[^\s]+\s+)?define\s+(?P<name>[^\s]+)"), + r"typedef struct vl_api_\g<name>_t"), + + # For every "brief" statement at the start of a comment block, add an + # xref with whatever is on the same line. This gives us an index page + # with all the API methods in one place. + # XXX Commented out for now; works but duplicates the brief text in the + # struct documentation + # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)(\*/)$"), + # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'), # capture inline comment close + # (re.compile(r"/\*\*\s*(?P<b>[\\@]brief)\s+(?P<c>.+)$"), + # r'/** @xrefitem api "" "VPP API" \g<c> \g<b> \g<c>'), + + # Since structs don't have params, replace @param with @tparam + ( re.compile("[\\@]param\\b"), "@tparam"), +] + +with open(sys.argv[1]) as fd: + for line in fd: + str = line[:-1] # strip \n + for p in patterns: + str = p[0].sub(p[1], str) + sys.stdout.write(str+"\n") diff --git a/docs/_scripts/filter_c.py b/docs/_scripts/filter_c.py new file mode 100755 index 00000000000..897f9f6d0b3 --- /dev/null +++ b/docs/_scripts/filter_c.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Filter for .c files to make various preprocessor tricks Doxygenish + +import os, sys, re + +if len(sys.argv) < 2: + sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0])) + sys.exit(1) + +replace_patterns = [ + # Search for VLIB_CLI_COMMAND, extract its parameters and add a docblock for it + ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_cli_command_t \g<name>"), + ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_cli_command_t \g<name>"), + + # Search for VLIB_REGISTER_NODE, extract its parameters and add a docblock for it + ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>) */ vlib_node_registration_t \g<name>"), + ( re.compile("(?P<m>VLIB_REGISTER_NODE)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<qual>[^)]*)[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>) */ \g<qual> vlib_node_registration_t \g<name>"), + + # Search for VLIB_INIT_FUNCTION, extract its parameter and add a docblock for it + ( re.compile("(?P<m>VLIB_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"), + r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"), + ( re.compile("(?P<m>VLIB_DECLARE_INIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)[)]"), + r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ vlib_init_function_t * _vlib_init_function_\g<name>"), + + # Search for VLIB_LOOP_ENTER_FUNCTION, extract the parameters and add a docblock for it + ( re.compile("(?P<m>VLIB_MAIN_LOOP_ENTER_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), + r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_enter_\g<name>"), + ( re.compile("(?P<m>VLIB_MAIN_LOOP_EXIT_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), + r"/** @brief (@em constructor) \g<m> (@ref \g<name>) */ _vlib_main_loop_exit_\g<name>"), + + # Search for VLIB_CONFIG_FUNCTION, extract the parameters and add a docblock for it + ( re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"), + ( re.compile("(?P<m>VLIB_EARLY_CONFIG_FUNCTION)\s*[(](?P<name>[a-zA-Z0-9_]+),\s*(?P<n>\"[^\"]+\")(,[^)]*)?[)]"), + r"/** @brief (@em constructor) \g<m> (\g<name>, \g<n>) */ vlib_config_function_runtime_t _vlib_config_function_\g<name>"), + + # Search for "format_thing" and "unformat_thing" when used as a function pointer and add parens + ( re.compile("(?P<pre>(^|,)\s*)(?P<name>(un)?format_[a-zA-Z0-9_]+)(?P<post>\s*(,|$))"), + r"\g<pre>\g<name>()\g<post>" ), + + # Search for CLIB_PAD_FROM_TO(...); and replace with padding + # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)] + ( re.compile("(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+),\s*(?P<to>[^)]+)[)]"), + r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]" ), + +] + + +filename = sys.argv[1] +cwd = os.getcwd() +if filename[0:len(cwd)] == cwd: + filename = filename[len(cwd):] + if filename[0] == "/": + filename = filename[1:] + +with open(filename) as fd: + line_num = 0 + + for line in fd: + line_num += 1 + str = line[:-1] # filter \n + + # Look for search/replace patterns + for p in replace_patterns: + str = p[0].sub(p[1], str) + + sys.stdout.write(str+"\n") + +# All done diff --git a/docs/_scripts/filter_h.py b/docs/_scripts/filter_h.py new file mode 100755 index 00000000000..0891fa708e1 --- /dev/null +++ b/docs/_scripts/filter_h.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Filter for .c files to make various preprocessor tricks Doxygenish + +import os +import re +import sys + +if len(sys.argv) < 2: + sys.stderr.write("Usage: %s <filename>\n" % (sys.argv[0])) + sys.exit(1) + +replace_patterns = [ + # Search for CLIB_PAD_FROM_TO(...); and replace with padding + # #define CLIB_PAD_FROM_TO(from,to) u8 pad_##from[(to) - (from)] + (re.compile(r"(?P<m>CLIB_PAD_FROM_TO)\s*[(](?P<from>[^,]+)," + r"\s*(?P<to>[^)]+)[)]"), + r"/** Padding. */ u8 pad_\g<from>[(\g<to>) - (\g<from>)]"), + +] + + +filename = sys.argv[1] +cwd = os.getcwd() +if filename[0:len(cwd)] == cwd: + filename = filename[len(cwd):] + if filename[0] == "/": + filename = filename[1:] + +with open(filename) as fd: + line_num = 0 + + for line in fd: + line_num += 1 + str = line[:-1] # filter \n + + # Look for search/replace patterns + for p in replace_patterns: + str = p[0].sub(p[1], str) + + sys.stdout.write(str+"\n") + +# All done diff --git a/docs/_scripts/includes_renderer.py b/docs/_scripts/includes_renderer.py new file mode 100644 index 00000000000..6bd501d83ff --- /dev/null +++ b/docs/_scripts/includes_renderer.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020. Vinci Consulting Corp. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import inspect +import os +import re +import sys + + +class ContentRenderer: + def __init__(self, ws_root, output_dir): + self.ws_root = ws_root + self.output_dir = output_dir + + def plugin_dir(self): + return os.path.join(self.ws_root, "src/plugins") + + def render(self): + raise NotImplementedError + + +class PluginRenderer(ContentRenderer): + + def _render_entry(self, output_file, entry): + description = "<no-description-found>" + # we use glob because a plugin can (ioam for now) + # define the plugin definition in + # a further subdirectory. + path = os.path.join(self.plugin_dir(), entry.name, '**') + for f in glob.iglob(path, recursive=True): + if not f.endswith('.c'): + continue + with open(f, "r", encoding="utf-8") as src: + for match in self.regex.finditer(src.read()): + description = "%s" % (match.group(1)) + + output_file.write(f"* {entry.name} - {description}\n") + + def render(self): + pattern = r'VLIB_PLUGIN_REGISTER\s?\(\)\s*=\s*{.*\.description\s?=\s?"([^"]*)".*};' # noqa: 501 + self.regex = re.compile(pattern, re.MULTILINE | re.DOTALL) + fname = os.path.join(self.output_dir, "plugin_list.inc") + with open(fname, "w") as output_file: + with os.scandir(self.plugin_dir()) as pdir: + for entry in sorted(pdir, key=lambda entry: entry.name): + if not entry.name.startswith('.') and entry.is_dir(): + self._render_entry(output_file, entry) + + +renderers = [PluginRenderer] + + +def main(): + if len(sys.argv) != 3: + print("You need to pass WS_ROOT and OUTPUT_DIR") + exit(1) + + print("rendering dynamic includes...") + for renderer in renderers: + renderer(*sys.argv[1:]).render() + print("done.") + + +if __name__ == "__main__": + main() diff --git a/docs/_scripts/prepare-for-site.sh b/docs/_scripts/prepare-for-site.sh new file mode 100755 index 00000000000..15888ba6e83 --- /dev/null +++ b/docs/_scripts/prepare-for-site.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ! -d "docs" ]; then + echo "This script is meant to be run from the root directory" + exit 1; +fi + +for f in $(find ./docs -type l) +do + target=$(readlink $f) + rm $f + cp $(dirname $f)/$target $(dirname $f)/$(basename $target) + echo "Replaced symlink $f" +done + +echo "Cleaning doc build directory" +make docs-clean + diff --git a/docs/_scripts/publish-docs.sh b/docs/_scripts/publish-docs.sh new file mode 100755 index 00000000000..026ab2f3907 --- /dev/null +++ b/docs/_scripts/publish-docs.sh @@ -0,0 +1,85 @@ +#!/bin/bash -ex + +# publish-docs.sh +# +# This sccript is used to publish the VPP User documents to +# the FD.io Site. +# +# Arguments: +# +# $1: The main site repo user name +# $2: The release branch name for example 1908, 1904 etc. + +# Some basic checks +if [ ! -d "docs" ]; then + echo "This script is meant to be run from the root directory." + exit 1; +fi + +if [ "$#" -ne 2 ]; then + echo "Please specify the site username and branch." + exit 1; +fi + +# Get the workspace root +WS_ROOT=$PWD + +# Get the VPP branch and username +SITE_USERNAME=$1 +VPP_BRANCH=$2 + +#Build the docs +make docs-clean +make docs-venv +make docs + +# Clone the site repo +rm -fr site +rm -fr sphinx_env +git clone ssh://git@github.com/$SITE_USERNAME/site +cd site +git submodule update --init --recursive +git remote add upstream ssh://git@github.com/FDio/site +git remote -v +git fetch upstream +git merge -m "Publish the Docs" upstream/master + +# Get the version +VERSION=`source $WS_ROOT/src/scripts/version` +VERSION=${VERSION/"~"/"-"} + +# Copy the files to the appropriate directory +SRC_DIR=../docs/_build/html/. +if [ "$VPP_BRANCH" == "master" ] +then + TARGET_DIR=./static/docs/vpp/master + rm -fr $TARGET_DIR +else + TARGET_DIR=./static/docs/vpp/v$VPP_BRANCH + rm -fr $TARGET_DIR + mkdir -p $TARGET_DIR + VERSION=v$VPP_BRANCH + rm ./static/docs/vpp/latest + ln -s $VERSION ./static/docs/vpp/latest +fi + +# Create a branch for the commit +git checkout -b $VERSION +git branch + +# Copy the docs +cp -r $SRC_DIR $TARGET_DIR + +# Create the feature list +pushd .. +source ./docs/venv/bin/activate +find . -name FEATURE.yaml | ./src/scripts/fts.py --markdown > site/content/vppProject/vppfeatures/features.md +deactivate +popd + +# Push the new docs +git add "*" +git commit -s -m "Publish docs from VPP $VERSION" +git push origin "$VERSION" + +exit 0 diff --git a/docs/_scripts/siphon-generate b/docs/_scripts/siphon-generate new file mode 100755 index 00000000000..9b69c52cf17 --- /dev/null +++ b/docs/_scripts/siphon-generate @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Looks for preprocessor macros with struct initializers and siphons them +# off into another file for later parsing; ostensibly to generate +# documentation from struct initializer data. + +import argparse +import logging +import os + +import siphon + +DEFAULT_LOGFILE = None +DEFAULT_LOGLEVEL = "info" +DEFAULT_OUTPUT = "build-root/docs/siphons" +DEFAULT_PREFIX = os.getcwd() + +ap = argparse.ArgumentParser() +ap.add_argument("--log-file", default=DEFAULT_LOGFILE, + help="Log file [%s]" % DEFAULT_LOGFILE) +ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL, + choices=["debug", "info", "warning", "error", "critical"], + help="Logging level [%s]" % DEFAULT_LOGLEVEL) + +ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT, + help="Output directory for .siphon files [%s]" % + DEFAULT_OUTPUT) +ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX, + help="Prefix to strip from input pathnames [%s]" % + DEFAULT_PREFIX) +ap.add_argument("input", nargs='+', metavar="input_file", + help="Input C source files") +args = ap.parse_args() + +logging.basicConfig(filename=args.log_file, + level=getattr(logging, args.log_level.upper(), None)) +log = logging.getLogger("siphon_generate") + + +generate = siphon.generate.Generate(output_directory=args.output, + input_prefix=args.input_prefix) + +# Pre-process file names in case they indicate a file with +# a list of files +files = [] +for filename in args.input: + if filename.startswith('@'): + with open(filename[1:], 'r') as fp: + lines = fp.readlines() + for line in lines: + file = line.strip() + if file not in files: + files.append(file) + lines = None + else: + if filename not in files: + files.append(filename) + +# Iterate all the input files we've been given +for filename in files: + generate.parse(filename) + +# Write the extracted data +generate.deliver() + +# All done diff --git a/docs/_scripts/siphon-process b/docs/_scripts/siphon-process new file mode 100755 index 00000000000..cbee1e90786 --- /dev/null +++ b/docs/_scripts/siphon-process @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Filter for .siphon files that are generated by other filters. +# The idea is to siphon off certain initializers so that we can better +# auto-document the contents of that initializer. + +import argparse +import logging +import os +import sys + +import siphon + +DEFAULT_LOGFILE = None +DEFAULT_LOGLEVEL = "info" +DEFAULT_SIPHON = "clicmd" +DEFAULT_FORMAT = "markdown" +DEFAULT_OUTPUT = None +DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates" +DEFAULT_OUTPUT_DIR = os.path.dirname(__file__) + "/siphon_docs" +DEFAULT_REPO_LINK = "https://github.com/FDio/vpp/blob/master/" + +ap = argparse.ArgumentParser() +ap.add_argument("--log-file", default=DEFAULT_LOGFILE, + help="Log file [%s]" % DEFAULT_LOGFILE) +ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL, + choices=["debug", "info", "warning", "error", "critical"], + help="Logging level [%s]" % DEFAULT_LOGLEVEL) + +ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON, + choices=siphon.process.siphons.keys(), + help="Siphon type to process [%s]" % DEFAULT_SIPHON) +ap.add_argument("--format", '-f', default=DEFAULT_FORMAT, + choices=siphon.process.formats.keys(), + help="Output format to generate [%s]" % DEFAULT_FORMAT) +ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT, + help="Output file (uses stdout if not defined) [%s]" % + DEFAULT_OUTPUT) +ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES, + help="Path to render templates directory [%s]" % + DEFAULT_TEMPLATES) +ap.add_argument("--outdir", metavar="directory", default=DEFAULT_OUTPUT_DIR, + help="Path to output rendered parts [%s]" % + DEFAULT_OUTPUT_DIR) +ap.add_argument("--repolink", metavar="repolink", default=DEFAULT_REPO_LINK, + help="Link to public repository [%s]" % + DEFAULT_REPO_LINK) +ap.add_argument("input", nargs='+', metavar="input_file", + help="Input .siphon files") +args = ap.parse_args() + +logging.basicConfig(filename=args.log_file, + level=getattr(logging, args.log_level.upper(), None)) +log = logging.getLogger("siphon_process") + +# Determine where to send the generated output +if args.output is None: + out = sys.stdout +else: + out = open(args.output, "w+") + +# Get our processor +klass = siphon.process.siphons[args.type] +processor = klass( + template_directory=args.templates, + format=args.format, + outdir=args.outdir, + repository_link=args.repolink +) + +# Load the input files +processor.load_json(args.input) + +# Process the data +processor.process(out=out) + +# All done diff --git a/docs/_scripts/siphon/__init__.py b/docs/_scripts/siphon/__init__.py new file mode 100644 index 00000000000..f6417314d82 --- /dev/null +++ b/docs/_scripts/siphon/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Siphon classes + +from . import generate +from . import generate_clicmd +from . import generate_syscfg + +from . import parsers +from . import process +from . import process_clicmd +from . import process_syscfg diff --git a/docs/_scripts/siphon/generate.py b/docs/_scripts/siphon/generate.py new file mode 100644 index 00000000000..2ae5a1b6f1b --- /dev/null +++ b/docs/_scripts/siphon/generate.py @@ -0,0 +1,306 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate .siphon source fragments for later processing + +import json +import logging +import os +import re + +"""List of (regexp, siphon_name) tuples for matching the start of C + initializer blocks in source files. Each siphon class registers + themselves on this list.""" +siphon_patterns = [] + +class Generate(object): + """Matches a siphon comment block start""" + siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$") + + """Matches a siphon comment block stop""" + siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$") + + """Siphon block directive delimiter""" + siphon_block_delimiter = "%%" + + """Matches a siphon block directive such as + '%clicmd:group_label Debug CLI%'""" + siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \ + (siphon_block_delimiter, siphon_block_delimiter)) + + """Matches the start of an initializer block""" + siphon_initializer = re.compile("\s*=") + + """Collated output for each siphon""" + output = None + + """Directory prefix to strip from input filenames to keep things tidy.""" + input_prefix = None + + """List of known siphons""" + known_siphons = None + + """Logging handler""" + log = None + + + def __init__(self, output_directory, input_prefix): + super(Generate, self).__init__() + self.log = logging.getLogger("siphon.generate") + + # Build a list of known siphons + self.known_siphons = [] + for item in siphon_patterns: + siphon = item[1] + if siphon not in self.known_siphons: + self.known_siphons.append(siphon) + + # Setup information for siphons we know about + self.output = {} + for siphon in self.known_siphons: + self.output[siphon] = { + "file": "%s/%s.siphon" % (output_directory, siphon), + "global": {}, + "items": [], + } + + self.input_prefix = input_prefix + + + """ + count open and close braces in str + return (0, index) when braces were found and count becomes 0. + index indicates the position at which the last closing brace was + found. + return (-1, -1) if a closing brace is found before any opening one. + return (count, -1) if not all opening braces are closed, count is the + current depth + """ + def count_braces(self, str, count=0, found=False): + for index in range(0, len(str)): + if str[index] == '{': + count += 1; + found = True + elif str[index] == '}': + if count == 0: + # means we never found an open brace + return (-1, -1) + count -= 1; + + if count == 0 and found: + return (count, index) + + return (count, -1) + + def parse(self, filename): + # Strip the current directory off the start of the + # filename for brevity + if filename[0:len(self.input_prefix)] == self.input_prefix: + filename = filename[len(self.input_prefix):] + if filename[0] == "/": + filename = filename[1:] + + # Work out the abbreviated directory name + directory = os.path.dirname(filename) + if directory[0:2] == "./": + directory = directory[2:] + elif directory[0:len(self.input_prefix)] == self.input_prefix: + directory = directory[len(self.input_prefix):] + if directory[0] == "/": + directory = directory[1:] + + # Open the file and explore its contents... + self.log.info("Siphoning from %s." % filename) + directives = {} + with open(filename) as fd: + siphon = None + close_siphon = None + siphon_block = "" + in_block = False + line_num = 0 + siphon_line = 0 + + for line in fd: + line_num += 1 + str = line[:-1] # filter \n + + """See if there is a block directive and if so extract it""" + def process_block_directive(str, directives): + m = self.siphon_block_directive.search(str) + if m is not None: + k = m.group(2) + v = m.group(3).strip() + directives[k] = v + # Return only the parts we did not match + return str[0:m.start(1)] + str[m.end(4):] + + return str + + def process_block_prefix(str): + if str.startswith(" * "): + str = str[3:] + elif str == " *": + str = "" + return str + + if not in_block: + # See if the line contains the start of a siphon doc block + m = self.siphon_block_start.search(str) + if m is not None: + in_block = True + t = m.group(1) + + # Now check if the block closes on the same line + m = self.siphon_block_stop.search(t) + if m is not None: + t = m.group(1) + in_block = False + + # Check for directives + t = process_block_directive(t, directives) + + # Filter for normal comment prefixes + t = process_block_prefix(t) + + # Add what is left + siphon_block += t + + # Skip to next line + continue + + else: + # Check to see if we have an end block marker + m = self.siphon_block_stop.search(str) + if m is not None: + in_block = False + t = m.group(1) + else: + t = str + + # Check for directives + t = process_block_directive(t, directives) + + # Filter for normal comment prefixes + t = process_block_prefix(t) + + # Add what is left + siphon_block += t + "\n" + + # Skip to next line + continue + + + if siphon is None: + # Look for blocks we need to siphon + for p in siphon_patterns: + if p[0].match(str): + siphon = [ p[1], str + "\n", 0 ] + siphon_line = line_num + + # see if we have an initializer + m = self.siphon_initializer.search(str) + if m is not None: + # count the braces on this line + (count, index) = \ + self.count_braces(str[m.start():]) + siphon[2] = count + # TODO - it's possible we have the + # initializer all on the first line + # we should check for it, but also + # account for the possibility that + # the open brace is on the next line + #if count == 0: + # # braces balanced + # close_siphon = siphon + # siphon = None + else: + # no initializer: close the siphon right now + close_siphon = siphon + siphon = None + else: + # See if we should end the siphon here - do we have + # balanced braces? + (count, index) = self.count_braces(str, + count=siphon[2], found=True) + if count == 0: + # braces balanced - add the substring and + # close the siphon + siphon[1] += str[:index+1] + ";\n" + close_siphon = siphon + siphon = None + else: + # add the whole string, move on + siphon[2] = count + siphon[1] += str + "\n" + + if close_siphon is not None: + # Write the siphoned contents to the right place + siphon_name = close_siphon[0] + + # Copy directives for the file + details = {} + for key in directives: + if ":" in key: + (sn, label) = key.split(":") + if sn == siphon_name: + details[label] = directives[key] + else: + details[key] = directives[key] + + # Copy details for this block + details['file'] = filename + details['directory'] = directory + details['line_start'] = siphon_line + details['line_end'] = line_num + details['siphon_block'] = siphon_block.strip() + details["block"] = close_siphon[1] + + # Store the item + self.output[siphon_name]['items'].append(details) + + # All done + close_siphon = None + siphon_block = "" + + # Update globals + for key in directives.keys(): + if ':' not in key: + continue + + if filename.endswith("/dir.dox"): + # very special! use the parent directory name + l = directory + else: + l = filename + + (sn, label) = key.split(":") + + if sn not in self.output: + self.output[sn] = {} + if 'global' not in self.output[sn]: + self.output[sn]['global'] = {} + if l not in self.output[sn]['global']: + self.output[sn]['global'][l] = {} + + self.output[sn]['global'][l][label] = directives[key] + + def deliver(self): + # Write out the data + for siphon in self.output.keys(): + self.log.info("Saving siphon data %s." % siphon) + s = self.output[siphon] + with open(s['file'], "a") as fp: + json.dump(s, fp, + separators=(',', ': '), indent=4, sort_keys=True) + diff --git a/docs/_scripts/siphon/generate_clicmd.py b/docs/_scripts/siphon/generate_clicmd.py new file mode 100644 index 00000000000..6d24aaf4926 --- /dev/null +++ b/docs/_scripts/siphon/generate_clicmd.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from . import generate + +# Register our regexp +generate.siphon_patterns.append(( + re.compile("(?P<m>VLIB_CLI_COMMAND)\s*" + "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), + "clicmd" +)) diff --git a/docs/_scripts/siphon/generate_syscfg.py b/docs/_scripts/siphon/generate_syscfg.py new file mode 100644 index 00000000000..52c802e5752 --- /dev/null +++ b/docs/_scripts/siphon/generate_syscfg.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from . import generate + +# Register our regexp +generate.siphon_patterns.append(( + re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*" + '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'), + "syscfg" +)) diff --git a/docs/_scripts/siphon/parsers.py b/docs/_scripts/siphon/parsers.py new file mode 100644 index 00000000000..162205de4ca --- /dev/null +++ b/docs/_scripts/siphon/parsers.py @@ -0,0 +1,150 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import html +import pyparsing as pp + +# Some useful primitives +ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_") +intNum = pp.Word(pp.nums) +hexNum = pp.Literal("0x") + pp.Word(pp.hexnums) +octalNum = pp.Literal("0") + pp.Word("01234567") +integer = (hexNum | octalNum | intNum) + \ + pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L")) +floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f")) +char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'") +arrayIndex = integer | ident + +lbracket = pp.Literal("(").suppress() +rbracket = pp.Literal(")").suppress() +lbrace = pp.Literal("{").suppress() +rbrace = pp.Literal("}").suppress() +comma = pp.Literal(",").suppress() +equals = pp.Literal("=").suppress() +dot = pp.Literal(".").suppress() +semicolon = pp.Literal(";").suppress() + +# initializer := { [member = ] (variable | expression | { initializer } ) } +typeName = ident +varName = ident +typeSpec = pp.Optional("unsigned") + \ + pp.oneOf("int long short float double char u8 i8 void") + \ + pp.Optional(pp.Word("*"), default="") +typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress() + +string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"', + escChar='\\', multiline=True)), adjacent=False) +literal = pp.Optional(typeCast) + (integer | floatNum | char | string) +var = pp.Combine(pp.Optional(typeCast) + varName + + pp.Optional("[" + arrayIndex + "]")) + +# This could be more complete, but suffices for our uses +expr = (literal | var) + +"""Parse and render a block of text into a Python dictionary.""" +class Parser(object): + """Compiled PyParsing BNF""" + _parser = None + + def __init__(self): + super(Parser, self).__init__() + self._parser = self.BNF() + + def BNF(self): + raise NotImplementedError + + def item(self, item): + raise NotImplementedError + + def parse(self, input): + item = self._parser.parseString(input).asList() + return self.item(item) + + +"""Parser for function-like macros - without the closing semi-colon.""" +class ParserFunctionMacro(Parser): + def BNF(self): + # VLIB_CONFIG_FUNCTION (unix_config, "unix") + macroName = ident + params = pp.Group(pp.ZeroOrMore(expr + comma) + expr) + macroParams = lbracket + params + rbracket + + return macroName + macroParams + + def item(self, item): + r = { + "macro": item[0], + "name": item[1][1], + "function": item[1][0], + } + + return r + + +"""Parser for function-like macros with a closing semi-colon.""" +class ParseFunctionMacroStmt(ParserFunctionMacro): + def BNF(self): + # VLIB_CONFIG_FUNCTION (unix_config, "unix"); + function_macro = super(ParseFunctionMacroStmt, self).BNF() + mi = function_macro + semicolon + mi.ignore(pp.cppStyleComment) + + return mi + + +""" +Parser for our struct initializers which are composed from a +function-like macro, equals sign, and then a normal C struct initializer +block. +""" +class MacroInitializer(ParserFunctionMacro): + def BNF(self): + # VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = { + # .path = "show sr tunnel", + # .short_help = "show sr tunnel [name <sr-tunnel-name>]", + # .function = show_sr_tunnel_fn, + # }; + cs = pp.Forward() + + + member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"), + adjacent=False) + value = (expr | cs) + + entry = pp.Group(pp.Optional(member + equals, default="") + value) + entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \ + (pp.ZeroOrMore(entry + comma)) + + cs << (lbrace + entries + rbrace) + + macroName = ident + params = pp.Group(pp.ZeroOrMore(expr + comma) + expr) + macroParams = lbracket + params + rbracket + + function_macro = super(MacroInitializer, self).BNF() + mi = function_macro + equals + pp.Group(cs) + semicolon + mi.ignore(pp.cppStyleComment) + + return mi + + def item(self, item): + r = { + "macro": item[0], + "name": item[1][0], + "params": item[2], + "value": {}, + } + + for param in item[2]: + r["value"][param[0]] = html.escape(param[1]) + + return r diff --git a/docs/_scripts/siphon/process.py b/docs/_scripts/siphon/process.py new file mode 100644 index 00000000000..e3a70152487 --- /dev/null +++ b/docs/_scripts/siphon/process.py @@ -0,0 +1,407 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generation template class + +import html.parser +import json +import logging +import os +import sys +import re + +import jinja2 + +# Classes register themselves in this dictionary +"""Mapping of known processors to their classes""" +siphons = {} + +"""Mapping of known output formats to their classes""" +formats = {} + + +class Siphon(object): + """Generate rendered output for siphoned data.""" + + # Set by subclasses + """Our siphon name""" + name = None + + # Set by subclasses + """Name of an identifier used by this siphon""" + identifier = None + + # Set by subclasses + """The pyparsing object to use to parse with""" + _parser = None + + """The input data""" + _cmds = None + + """Group key to (directory,file) mapping""" + _group = None + + """Logging handler""" + log = None + + """Directory to look for siphon rendering templates""" + template_directory = None + + """Directory to output parts in""" + outdir = None + + """Template environment, if we're using templates""" + _tplenv = None + + def __init__(self, template_directory, format, outdir, repository_link): + super(Siphon, self).__init__() + self.log = logging.getLogger("siphon.process.%s" % self.name) + + # Get our output format details + fmt_klass = formats[format] + fmt = fmt_klass() + self._format = fmt + + # Sort out the template search path + def _tpldir(name): + return os.sep.join((template_directory, fmt.name, name)) + + self.template_directory = template_directory + searchpath = [ + _tpldir(self.name), + _tpldir("default"), + ] + self.outdir = outdir + loader = jinja2.FileSystemLoader(searchpath=searchpath) + self._tplenv = jinja2.Environment( + loader=loader, + trim_blocks=True, + autoescape=False, + keep_trailing_newline=True) + + # Convenience, get a reference to the internal escape and + # unescape methods in html.parser. These then become + # available to templates to use, if needed. + self._h = html.parser.HTMLParser() + self.escape = html.escape + self.unescape = html.unescape + + # TODO: customize release + self.repository_link = repository_link + + # Output renderers + + """Returns an object to be used as the sorting key in the item index.""" + def index_sort_key(self, group): + return group + + """Returns a string to use as the header at the top of the item index.""" + def index_header(self): + return self.template("index_header") + + """Returns the string fragment to use for each section in the item + index.""" + def index_section(self, group): + return self.template("index_section", group=group) + + """Returns the string fragment to use for each entry in the item index.""" + def index_entry(self, meta, item): + return self.template("index_entry", meta=meta, item=item) + + """Returns an object, typically a string, to be used as the sorting key + for items within a section.""" + def item_sort_key(self, item): + return item['name'] + + """Returns a key for grouping items together.""" + def group_key(self, directory, file, macro, name): + _global = self._cmds['_global'] + + if file in _global and 'group_label' in _global[file]: + self._group[file] = (directory, file) + return file + + self._group[directory] = (directory, None) + return directory + + """Returns a key for identifying items within a grouping.""" + def item_key(self, directory, file, macro, name): + return name + + """Returns a string to use as the header when rendering the item.""" + def item_header(self, group): + return self.template("item_header", group=group) + + """Returns a string to use as the body when rendering the item.""" + def item_format(self, meta, item): + return self.template("item_format", meta=meta, item=item) + + """Returns a string to use as the label for the page reference.""" + def page_label(self, group): + return "_".join(( + self.name, + self.sanitize_label(group) + )) + + """Returns a title to use for a page.""" + def page_title(self, group): + _global = self._cmds['_global'] + (directory, file) = self._group[group] + + if file and file in _global and 'group_label' in _global[file]: + return _global[file]['group_label'] + + if directory in _global and 'group_label' in _global[directory]: + return _global[directory]['group_label'] + + return directory + + """Returns a string to use as the label for the section reference.""" + def item_label(self, group, item): + return "__".join(( + self.name, + item + )) + + """Label sanitizer; for creating Doxygen references""" + def sanitize_label(self, value): + return value.replace(" ", "_") \ + .replace("/", "_") \ + .replace(".", "_") + + """Template processor""" + def template(self, name, **kwargs): + tpl = self._tplenv.get_template(name + self._format.extension) + return tpl.render( + this=self, + **kwargs) + + # Processing methods + + """Parse the input file into a more usable dictionary structure.""" + def load_json(self, files): + self._cmds = {} + self._group = {} + + line_num = 0 + line_start = 0 + for filename in files: + filename = os.path.relpath(filename) + self.log.info("Parsing items in file \"%s\"." % filename) + data = None + with open(filename, "r") as fd: + data = json.load(fd) + + self._cmds['_global'] = data['global'] + + # iterate the items loaded and regroup it + for item in data["items"]: + try: + o = self._parser.parse(item['block']) + except Exception: + self.log.error("Exception parsing item: %s\n%s" + % (json.dumps(item, separators=(',', ': '), + indent=4), + item['block'])) + raise + + # Augment the item with metadata + o["meta"] = {} + for key in item: + if key == 'block': + continue + o['meta'][key] = item[key] + + # Load some interesting fields + directory = item['directory'] + file = item['file'] + macro = o["macro"] + name = o["name"] + + # Generate keys to group items by + group_key = self.group_key(directory, file, macro, name) + item_key = self.item_key(directory, file, macro, name) + + if group_key not in self._cmds: + self._cmds[group_key] = {} + + self._cmds[group_key][item_key] = o + + """Iterate over the input data, calling render methods to generate the + output.""" + def process(self, out=None): + + if out is None: + out = sys.stdout + + # Accumulated body contents + contents = "" + + # Write the header for this siphon type + out.write(self.index_header()) + + # Sort key helper for the index + def group_sort_key(group): + return self.index_sort_key(group) + + # Iterate the dictionary and process it + for group in sorted(self._cmds.keys(), key=group_sort_key): + if group.startswith('_'): + continue + + self.log.info("Processing items in group \"%s\" (%s)." % + (group, group_sort_key(group))) + + # Generate the section index entry (write it now) + out.write(self.index_section(group)) + + # Generate the item header (save for later) + contents += self.item_header(group) + + def item_sort_key(key): + return self.item_sort_key(self._cmds[group][key]) + + for key in sorted(self._cmds[group].keys(), key=item_sort_key): + self.log.debug("--- Processing key \"%s\" (%s)." % + (key, item_sort_key(key))) + + o = self._cmds[group][key] + meta = { + "directory": o['meta']['directory'], + "file": o['meta']['file'], + "macro": o['macro'], + "name": o['name'], + "key": key, + "label": self.item_label(group, key), + } + + # Generate the index entry for the item (write it now) + out.write(self.index_entry(meta, o)) + + # Generate the item itself (save for later) + contents += self.item_format(meta, o) + + page_name = self.separate_page_names(group) + if page_name != "": + path = os.path.join(self.outdir, page_name) + with open(path, "w+") as page: + page.write(contents) + contents = "" + + # Deliver the accumulated body output + out.write(contents) + + def do_cliexstart(self, matchobj): + title = matchobj.group(1) + title = ' '.join(title.splitlines()) + content = matchobj.group(2) + content = re.sub(r"\n", r"\n ", content) + return "\n\n.. code-block:: console\n\n %s\n %s\n\n" % (title, content) + + def do_clistart(self, matchobj): + content = matchobj.group(1) + content = re.sub(r"\n", r"\n ", content) + return "\n\n.. code-block:: console\n\n %s\n\n" % content + + def do_cliexcmd(self, matchobj): + content = matchobj.group(1) + content = ' '.join(content.splitlines()) + return "\n\n.. code-block:: console\n\n %s\n\n" % content + + def process_list(self, matchobj): + content = matchobj.group(1) + content = self.reindent(content, 2) + return "@@@@%s\nBBBB" % content + + def process_special(self, s): + # ----------- markers to remove + s = re.sub(r"@cliexpar\s*", r"", s) + s = re.sub(r"@parblock\s*", r"", s) + s = re.sub(r"@endparblock\s*", r"", s) + s = re.sub(r"<br>", "", s) + # ----------- emphasis + # <b><em> + s = re.sub(r"<b><em>\s*", "``", s) + s = re.sub(r"\s*</b></em>", "``", s) + s = re.sub(r"\s*</em></b>", "``", s) + # <b> + s = re.sub(r"<b>\s*", "**", s) + s = re.sub(r"\s*</b>", "**", s) + # <code> + s = re.sub(r"<code>\s*", "``", s) + s = re.sub(r"\s*</code>", "``", s) + # <em> + s = re.sub(r"'?<em>\s*", r"``", s) + s = re.sub(r"\s*</em>'?", r"``", s) + # @c <something> + s = re.sub(r"@c\s(\S+)", r"``\1``", s) + # ----------- todos + s = re.sub(r"@todo[^\n]*", "", s) + s = re.sub(r"@TODO[^\n]*", "", s) + # ----------- code blocks + s = re.sub(r"@cliexcmd{(.+?)}", self.do_cliexcmd, s, flags=re.DOTALL) + s = re.sub(r"@cliexstart{(.+?)}(.+?)@cliexend", self.do_cliexstart, s, flags=re.DOTALL) + s = re.sub(r"@clistart(.+?)@cliend", self.do_clistart, s, flags=re.DOTALL) + # ----------- lists + s = re.sub(r"^\s*-", r"\n@@@@", s, flags=re.MULTILINE) + s = re.sub(r"@@@@(.*?)\n\n+", self.process_list, s, flags=re.DOTALL) + s = re.sub(r"BBBB@@@@", r"-", s) + s = re.sub(r"@@@@", r"-", s) + s = re.sub(r"BBBB", r"\n\n", s) + # ----------- Cleanup remains + s = re.sub(r"@cliexend\s*", r"", s) + return s + + def separate_page_names(self, group): + return "" + + # This push the given textblock <indent> spaces right + def reindent(self, s, indent): + ind = " " * indent + s = re.sub(r"\n", "\n" + ind, s) + return s + + # This aligns the given textblock left (no indent) + def noindent(self, s): + s = re.sub(r"\n[ \f\v\t]*", "\n", s) + return s + +class Format(object): + """Output format class""" + + """Name of this output format""" + name = None + + """Expected file extension of templates that build this format""" + extension = None + + +class FormatMarkdown(Format): + """Markdown output format""" + name = "markdown" + extension = ".md" + + +# Register 'markdown' +formats["markdown"] = FormatMarkdown + + +class FormatItemlist(Format): + """Itemlist output format""" + name = "itemlist" + extension = ".itemlist" + + +# Register 'itemlist' +formats["itemlist"] = FormatItemlist diff --git a/docs/_scripts/siphon/process_clicmd.py b/docs/_scripts/siphon/process_clicmd.py new file mode 100644 index 00000000000..bf270518ad1 --- /dev/null +++ b/docs/_scripts/siphon/process_clicmd.py @@ -0,0 +1,73 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate clicmd formatted output + +from . import process, parsers +import os + +class SiphonCLICMD(process.Siphon): + + name = "clicmd" + identifier = "VLIB_CLI_COMMAND" + + def __init__(self, *args, **kwargs): + super(SiphonCLICMD, self).__init__(*args, **kwargs) + self._parser = parsers.MacroInitializer() + + # Output renderers + + def separate_page_names(self, group): + return self.page_label(group) + ".rst" + + def index_sort_key(self, group): + _global = self._cmds['_global'] + if group not in self._group: + return group + (directory, file) = self._group[group] + + if file in _global and 'group_label' in _global[file]: + return _global[file]['group_label'] + + if directory in _global and 'group_label' in _global[directory]: + return _global[directory]['group_label'] + + return group + + def item_sort_key(self, item): + return item['value']['path'] + + def item_label(self, group, item): + return "_".join(( + self.name, + self.sanitize_label(self._cmds[group][item]['value']['path']) + )) + + def page_title(self, group): + _global = self._cmds['_global'] + (directory, file) = self._group[group] + + if file and file in _global and 'group_label' in _global[file]: + return _global[file]['group_label'] + + if directory in _global and 'group_label' in _global[directory]: + return _global[directory]['group_label'] + + file_ext = os.path.basename(directory) + fname, ext = os.path.splitext(file_ext) + return "%s cli reference" % fname.capitalize() + + +# Register our processor +process.siphons["clicmd"] = SiphonCLICMD diff --git a/docs/_scripts/siphon/process_syscfg.py b/docs/_scripts/siphon/process_syscfg.py new file mode 100644 index 00000000000..bccde2c153d --- /dev/null +++ b/docs/_scripts/siphon/process_syscfg.py @@ -0,0 +1,31 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate syscfg formatted output + +from . import process, parsers + + +class SiphonSYSCFG(process.Siphon): + + name = "syscfg" + identifier = "VLIB_CONFIG_FUNCTION" + + def __init__(self, *args, **kwargs): + super(SiphonSYSCFG, self).__init__(*args, **kwargs) + self._parser = parsers.ParseFunctionMacroStmt() + + +# Register our processor +process.siphons["syscfg"] = SiphonSYSCFG diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md new file mode 100644 index 00000000000..de517f160c1 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_entry.md @@ -0,0 +1,15 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#}
\ No newline at end of file diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md new file mode 100644 index 00000000000..86e5bb6c3e5 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_header.md @@ -0,0 +1,23 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +.. _cmdreference: + +Reference +========= + +.. toctree:: + :maxdepth: 2 + diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md new file mode 100644 index 00000000000..0da934ac938 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/clicmd/index_section.md @@ -0,0 +1,16 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + clis/{{ this.page_label(group) }} diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md new file mode 100644 index 00000000000..f1caa961f56 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_format.md @@ -0,0 +1,62 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% set v = item['value'] %} +{% if 'short_help' in v %} +{% set str = v['short_help'] %} +{% set str = this.unescape(str) %} + +{# Summary/usage #} +{{ item['value']['path'] }} +------------------------------------------------------------------------- + +.. code-block:: console + + {{ this.reindent(str, 4) }} + +{% endif %} +{% if 'long_help' in v %} +{% set long_help = v['long_help'] %} +{% set long_help = this.unescape(long_help) %} +{# This is seldom used and will likely be deprecated #} +{# Long help #} +.. code-block:: console + + {{ this.reindent(long_help, 4) }} + +{% endif %} +{% if 'siphon_block' in item['meta'] %} +{% set sb = item["meta"]["siphon_block"] %} +{% set sb = this.process_special(sb) %} +{% if sb %} +{# Extracted from the code in /*? ... ?*/ blocks #} + +{# Description #} + +{{ sb }} +{% endif %} +{% endif %} +{% if 'name' in meta or 'function' in v %} +{# Gives some developer-useful linking #} + +{% if "name" in meta %} +Declaration: ``{{ meta['name'] }}`` `{{ meta["file"] }} line {{ item["meta"]["line_start"] }} <{{ this.repository_link }}{{ meta["file"] }}#L{{ item["meta"]["line_start"] }}>`_ +{% endif %} +{% if "function" in v %} + +Implementation: ``{{ v["function"] }}`` +{% endif %} +{% endif %} + diff --git a/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md new file mode 100644 index 00000000000..20e4803ec2e --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/clicmd/item_header.md @@ -0,0 +1,20 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +.. _{{ this.page_label(group) }}: + +=============================================================== +{{ this.page_title(group) }} +=============================================================== diff --git a/docs/_scripts/siphon_templates/markdown/default/index_entry.md b/docs/_scripts/siphon_templates/markdown/default/index_entry.md new file mode 100644 index 00000000000..479dcdb2cc3 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/default/index_entry.md @@ -0,0 +1,16 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{{ "* [%s](@ref %s)" % (item["name"], meta["label"]) }} diff --git a/docs/_scripts/siphon_templates/markdown/default/index_section.md b/docs/_scripts/siphon_templates/markdown/default/index_section.md new file mode 100644 index 00000000000..3c9d2b474c5 --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/default/index_section.md @@ -0,0 +1,18 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +@subpage {{ this.page_label(group) }} + diff --git a/docs/_scripts/siphon_templates/markdown/default/item_format.md b/docs/_scripts/siphon_templates/markdown/default/item_format.md new file mode 100644 index 00000000000..ed1b1bf7eeb --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/default/item_format.md @@ -0,0 +1,16 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{{ raise NotImplementedError }} diff --git a/docs/_scripts/siphon_templates/markdown/default/item_header.md b/docs/_scripts/siphon_templates/markdown/default/item_header.md new file mode 100644 index 00000000000..0c21e51fa5d --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/default/item_header.md @@ -0,0 +1,18 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{{ "@page %s %s" % (this.page_label(group), this.page_title(group)) }} + diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md new file mode 100644 index 00000000000..f5a9816b99c --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/syscfg/index_header.md @@ -0,0 +1,111 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +# Startup Configuration {{'{#'}}syscfg} + +The VPP network stack comes with several configuration options that can be +provided either on the command line or in a configuration file. + +Specific applications built on the stack have been known to require a dozen +arguments, depending on requirements. This section describes commonly-used +options and parameters. + +You can find command-line argument parsers in the source code by searching for +instances of the `VLIB_CONFIG_FUNCTION` macro. The invocation +`VLIB_CONFIG_FUNCTION(foo_config, "foo")` will cause the function +`foo_config` to receive all the options and values supplied in a parameter +block named "`foo`", for example: `foo { arg1 arg2 arg3 ... }`. + +@todo Tell the nice people where this document lives so that the might +help improve it! + +## Command-line arguments + +Parameters are grouped by a section name. When providing more than one +parameter to a section all parameters for that section must be wrapped in +curly braces. + +``` +/usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 } +``` + +Which will produce output similar to this: + + <startup diagnostic messages> + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + vpp# <start-typing> + +When providing only one such parameter the braces are optional. For example, +the following command argument, `unix interactive` does not have braces: + +``` +/usr/bin/vpp unix interactive +``` + +The command line can be presented as a single string or as several; anything +given on the command line is concatenated with spaces into a single string +before parsing. + +VPP applications must be able to locate their own executable images. The +simplest way to ensure this will work is to invoke a VPP application by giving +its absolute path; for example: `/usr/bin/vpp <options>`. At startup, VPP +applications parse through their own ELF-sections (primarily) to make lists +of init, configuration, and exit handlers. + +When developing with VPP, in _gdb_ it's often sufficient to start an application +like this at the `(gdb)` prompt: + +``` +run unix interactive +``` + +## Configuration file + +It is also possible to supply parameters in a startup configuration file the +path of which is provided to the VPP application on its command line. + +The format of the configuration file is a simple text file with the same +content as the command line but with the benefit of being able to use newlines +to make the content easier to read. For example: + +``` +unix { + nodaemon + log /var/log/vpp/vpp.log + full-coredump + cli-listen localhost:5002 +} +api-trace { + on +} +dpdk { + dev 0000:03:00.0 +} +``` + +VPP is then instructed to load this file with the `-c` option: + +``` +/usr/bin/vpp -c /etc/vpp/startup.conf +``` + +## Index of startup command sections + +[TOC] + diff --git a/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md new file mode 100644 index 00000000000..53136115bbc --- /dev/null +++ b/docs/_scripts/siphon_templates/markdown/syscfg/item_format.md @@ -0,0 +1,42 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% set v = item['value'] %} +{{ "@section %s %s" % (meta['label'], item['name']) }} +{% if 'siphon_block' in item['meta'] %} +{% set sb = item["meta"]["siphon_block"] %} +{% if sb %} +{# Extracted from the code in /*? ... ?*/ blocks #} + +### Description + +{{ sb }} +{% endif %} +{% endif %} +{% if "name" in meta or "function" in item %} +{# Gives some developer-useful linking #} + +### Declaration and implementation +{% if "name" in meta %} + +{{ "Declaration: @ref %s (@ref %s line %d)" % + (meta['name'], meta["file"], item["meta"]["line_start"]) }} +{% endif %} +{% if "function" in item %} + +{{ "Implementation: @ref %s." % item["function"] }} +{% endif %} +{% endif %} + |