diff options
| author |   Chris Luke <chrisy@flirble.org> | 2016-09-12 08:55:13 -0400 |
|---|---|---|
| committer | Chris Luke <chrisy@flirble.org> | 2016-09-21 15:42:25 -0400 |
| commit | 90f52bf990791ea73479ffc50fc1eb3450de443a (patch) | |
| tree | ffcd6f5a94c4fc1a44ce9a3e088f18449007e3db | |
| parent | ce64b8e5b247149887caf77fd139d2a6880acbe6 (diff) | |
Refactor pre-Doxy siphon scripts; VPP-396
- Modularize the code to make the Siphon process easier to
maintain.
- Move much of the output rendering into Jinja2 templates.
- Add syscfg siphon type for startup config documentation.
- Add sample syscfg documentation.
- Add clicfg and syscfg preamble docs, adapted from their wiki pages.
- Fix sorting of CLI items across multiple directories.
Change-Id: Ib8288fe005adfea68ceed75a38ff8eba25d3cc79
Signed-off-by: Chris Luke <chrisy@flirble.org>
28 files changed, 1580 insertions, 660 deletions
diff --git a/doxygen/Makefile b/doxygen/Makefile index 0a69c2dfe21..97225f3cd57 100644 --- a/doxygen/Makefile +++ b/doxygen/Makefile @@ -16,14 +16,18 @@ # Build the documentation # +# Default target +.PHONY: all +all: doxygen + # These should be passed in by the root Makefile WS_ROOT ?= $(CURDIR)/.. BR ?= $(WS_ROOT)/build-root OS_ID ?= $(shell grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g') # Package dependencies -DOC_DEB_DEPENDS = doxygen graphviz python-pyparsing -DOC_RPM_DEPENDS = doxygen graphviz pyparsing +DOC_DEB_DEPENDS = doxygen graphviz python-pyparsing python-jinja2 +DOC_RPM_DEPENDS = doxygen graphviz pyparsing python-jinja2 # Doxygen configuration and our utility scripts DOXY_DIR ?= $(WS_ROOT)/doxygen @@ -104,19 +108,22 @@ SIPHON_OUTPUT ?= $(DOXY_OUTPUT)/siphon_docs EXTRA_DOXY_INPUT += $(SIPHON_OUTPUT) # All the siphon types we know about -SIPHONS ?= clicmd +SIPHONS ?= clicmd syscfg SIPHON_FILES = $(addprefix $(SIPHON_INPUT)/,$(addsuffix .siphon,$(SIPHONS))) SIPHON_DOCS = $(addprefix $(SIPHON_OUTPUT)/,$(addsuffix .md,$(SIPHONS))) -$(BR)/.doxygen-bootstrap.ok: +$(BR)/.doxygen-bootstrap.ok: Makefile @echo "Checking whether dependencies for Doxygen are installed..." ifeq ($(OS_ID),ubuntu) @set -e; inst=; \ for i in $(DOC_DEB_DEPENDS); do \ dpkg-query --show $$i >/dev/null 2>&1 || inst="$$inst $$i"; \ done; \ - if [ "$$inst" ]; then sudo apt-get $(CONFIRM) $(FORCE) install $$inst; fi + if [ "$$inst" ]; then \ + sudo apt-get update; \ + sudo apt-get $(CONFIRM) $(FORCE) install $$inst; \ + fi @if [ ! -s /usr/lib/graphviz/config6a ]; then \ echo "Rebuidlding system Graphviz configuration."; \ sudo dot -c; \ @@ -145,8 +152,12 @@ $(BR)/.doxygen-siphon.dep: Makefile # Include the source -> siphon dependencies -include $(BR)/.doxygen-siphon.dep +# Generate .siphon files that contain fragments of source file that +# relate to the siphons we support. .NOTPARALLEL: $(SIPHON_FILES) -$(SIPHON_FILES): $(DOXY_DIR)/siphon_generate.py $(BR)/.doxygen-bootstrap.ok +$(SIPHON_FILES): $(BR)/.doxygen-bootstrap.ok \ + $(DOXY_DIR)/siphon-generate \ + $(wildcard $(DOXY_DIR)/siphon/*.py) @rm -rf "$(SIPHON_INPUT)" "$(SIPHON_OUTPUT)" @mkdir -p "$(SIPHON_INPUT)" "$(SIPHON_OUTPUT)" @touch $(SIPHON_INPUT)/files @@ -159,23 +170,33 @@ $(SIPHON_FILES): $(DOXY_DIR)/siphon_generate.py $(BR)/.doxygen-bootstrap.ok >> $(SIPHON_INPUT)/files; \ done @echo "Generating siphons..." - @set -e; cd "$(WS_ROOT)"; $(DOXY_DIR)/siphon_generate.py \ + @set -e; \ + cd "$(WS_ROOT)"; \ + $(DOXY_DIR)/siphon-generate \ --output="$(SIPHON_INPUT)" \ "@$(SIPHON_INPUT)/files" - +# Process the .siphon source fragments and render them into doxygen flavored +# markdown documentation .DELETE_ON_ERROR: $(SIPHON_DOCS) -$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon $(DOXY_DIR)/siphon_process.py +$(SIPHON_OUTPUT)/%.md: $(SIPHON_INPUT)/%.siphon \ + $(DOXY_DIR)/siphon-process \ + $(wildcard $(DOXY_DIR)/siphon/*.py) \ + $(wildcard $(DOXY_DIR)/siphon_templates/*/*.md) @echo "Processing siphon from $(notdir $<)..." - @set -e; cd "$(WS_ROOT)"; \ - $(DOXY_DIR)/siphon_process.py --type=$(basename $(notdir $<)) \ - --output="$(SIPHON_OUTPUT)" $< > $@ + @set -e; \ + cd "$(WS_ROOT)"; \ + $(DOXY_DIR)/siphon-process \ + --type=$(basename $(notdir $<)) \ + --output="$@" \ + "$<" # This target can be used just to generate the siphoned docs .PHONY: doxygen-siphon doxygen-siphon: $(SIPHON_DOCS) # Generate the doxygen docs +.PHONY: doxygen doxygen: $(SIPHON_DOCS) @mkdir -p "$(DOXY_OUTPUT)" @echo "Running Doxygen..." @@ -189,6 +210,9 @@ doxygen: $(SIPHON_DOCS) VERSION="`git describe --tags --dirty`" \ doxygen $(DOXY_DIR)/doxygen.cfg +.PHONY: wipe-doxygen wipe-doxygen: rm -rf "$(BR)/docs" "$(BR)/.doxygen-siphon.d" +.PHONY: clean +clean: wipe-doxygen diff --git a/doxygen/dir.dox.sample b/doxygen/dir.dox.sample index 500fe595ae5..ccdd095abb2 100644 --- a/doxygen/dir.dox.sample +++ b/doxygen/dir.dox.sample @@ -27,3 +27,4 @@ This looks like a C file but it is not part of the build; it is purely for documentation. */ /*? %%clicmd:group_label CLI section description%% ?*/ +/*? %%syscfg:group_label Startup config section description%% ?*/ diff --git a/doxygen/doxygen.cfg b/doxygen/doxygen.cfg index 8844965b175..7675b693962 100644 --- a/doxygen/doxygen.cfg +++ b/doxygen/doxygen.cfg @@ -244,6 +244,9 @@ ALIASES += "cliexcmd{1}=@clistart<b>vpp# <em>\1</em></b>@cliend" ALIASES += "cliexstart{1}=@cliexcmd{\1}@clistart" ALIASES += "cliexend=@cliend" +## Formatting for config directives +ALIASES += "cfgcmd{2}=@par <code><pre>\1 \2</pre></code>" +ALIASES += "cfgcmd{1}=@par <code><pre>\1</pre></code>" # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding "class=itcl::class" diff --git a/doxygen/siphon-generate b/doxygen/siphon-generate new file mode 100755 index 00000000000..bdfd58d8bbb --- /dev/null +++ b/doxygen/siphon-generate @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Looks for preprocessor macros with struct initializers and siphons them +# off into another file for later parsing; ostensibly to generate +# documentation from struct initializer data. + +import os, sys, argparse, logging +import siphon + +DEFAULT_LOGFILE = None +DEFAULT_LOGLEVEL = "info" +DEFAULT_OUTPUT = "build-root/docs/siphons" +DEFAULT_PREFIX = os.getcwd() + +ap = argparse.ArgumentParser() +ap.add_argument("--log-file", default=DEFAULT_LOGFILE, + help="Log file [%s]" % DEFAULT_LOGFILE) +ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL, + choices=["debug", "info", "warning", "error", "critical"], + help="Logging level [%s]" % DEFAULT_LOGLEVEL) + +ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT, + help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT) +ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX, + help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX) +ap.add_argument("input", nargs='+', metavar="input_file", + help="Input C source files") +args = ap.parse_args() + +logging.basicConfig(filename=args.log_file, + level=getattr(logging, args.log_level.upper(), None)) +log = logging.getLogger("siphon_generate") + + +generate = siphon.generate.Generate(output_directory=args.output, + input_prefix=args.input_prefix) + +# Pre-process file names in case they indicate a file with +# a list of files +files = [] +for filename in args.input: + if filename.startswith('@'): + with open(filename[1:], 'r') as fp: + lines = fp.readlines() + for line in lines: + file = line.strip() + if file not in files: + files.append(file) + lines = None + else: + if filename not in files: + files.append(filename) + +# Iterate all the input files we've been given +for filename in files: + generate.parse(filename) + +# Write the extracted data +generate.deliver() + +# All done diff --git a/doxygen/siphon-process b/doxygen/siphon-process new file mode 100755 index 00000000000..ea9df96fe26 --- /dev/null +++ b/doxygen/siphon-process @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Filter for .siphon files that are generated by other filters. +# The idea is to siphon off certain initializers so that we can better +# auto-document the contents of that initializer. + +import os, sys, argparse, logging +import siphon + +DEFAULT_LOGFILE = None +DEFAULT_LOGLEVEL = "info" +DEFAULT_SIPHON ="clicmd" +DEFAULT_OUTPUT = None +DEFAULT_TEMPLATES = os.path.dirname(__file__) + "/siphon_templates" + +ap = argparse.ArgumentParser() +ap.add_argument("--log-file", default=DEFAULT_LOGFILE, + help="Log file [%s]" % DEFAULT_LOGFILE) +ap.add_argument("--log-level", default=DEFAULT_LOGLEVEL, + choices=["debug", "info", "warning", "error", "critical"], + help="Logging level [%s]" % DEFAULT_LOGLEVEL) + +ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON, + choices=siphon.process.siphons.keys(), + help="Siphon type to process [%s]" % DEFAULT_SIPHON) +ap.add_argument("--output", '-o', metavar="file", default=DEFAULT_OUTPUT, + help="Output file (uses stdout if not defined) [%s]" % DEFAULT_OUTPUT) +ap.add_argument("--templates", metavar="directory", default=DEFAULT_TEMPLATES, + help="Path to render templates directory [%s]" % DEFAULT_TEMPLATES) +ap.add_argument("input", nargs='+', metavar="input_file", + help="Input .siphon files") +args = ap.parse_args() + +logging.basicConfig(filename=args.log_file, + level=getattr(logging, args.log_level.upper(), None)) +log = logging.getLogger("siphon_process") + +# Determine where to send the generated output +if args.output is None: + out = sys.stdout +else: + out = open(args.output, "w+") + +# Get our processor +klass = siphon.process.siphons[args.type] +processor = klass(template_directory=args.templates) + +# Load the input files +processor.load_json(args.input) + +# Process the data +processor.process(out=out) + +# All done diff --git a/doxygen/siphon/__init__.py b/doxygen/siphon/__init__.py new file mode 100644 index 00000000000..437a1df1597 --- /dev/null +++ b/doxygen/siphon/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Siphon classes + +import generate +import generate_clicmd +import generate_syscfg + +import parsers +import process +import process_clicmd +import process_syscfg diff --git a/doxygen/siphon/generate.py b/doxygen/siphon/generate.py new file mode 100644 index 00000000000..d6b6faf34f7 --- /dev/null +++ b/doxygen/siphon/generate.py @@ -0,0 +1,304 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate .siphon source fragments for later processing + +import logging +import os, sys, re, json + +"""List of (regexp, siphon_name) tuples for matching the start of C + initializer blocks in source files. Each siphon class registers + themselves on tihs list.""" +siphon_patterns = [] + +class Generate(object): + """Matches a siphon comment block start""" + siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$") + + """Matches a siphon comment block stop""" + siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$") + + """Siphon block directive delimiter""" + siphon_block_delimiter = "%%" + + """Matches a siphon block directive such as + '%clicmd:group_label Debug CLI%'""" + siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \ + (siphon_block_delimiter, siphon_block_delimiter)) + + """Matches the start of an initializer block""" + siphon_initializer = re.compile("\s*=") + + """Collated output for each siphon""" + output = None + + """Directory prefix to strip from input filenames to keep things tidy.""" + input_prefix = None + + """List of known siphons""" + known_siphons = None + + """Logging handler""" + log = None + + + def __init__(self, output_directory, input_prefix): + super(Generate, self).__init__() + self.log = logging.getLogger("siphon.generate") + + # Build a list of known siphons + self.known_siphons = [] + for item in siphon_patterns: + siphon = item[1] + if siphon not in self.known_siphons: + self.known_siphons.append(siphon) + + # Setup information for siphons we know about + self.output = {} + for siphon in self.known_siphons: + self.output[siphon] = { + "file": "%s/%s.siphon" % (output_directory, siphon), + "global": {}, + "items": [], + } + + self.input_prefix = input_prefix + + + """ + count open and close braces in str + return (0, index) when braces were found and count becomes 0. + index indicates the position at which the last closing brace was + found. + return (-1, -1) if a closing brace is found before any opening one. + return (count, -1) if not all opening braces are closed, count is the + current depth + """ + def count_braces(self, str, count=0, found=False): + for index in range(0, len(str)): + if str[index] == '{': + count += 1; + found = True + elif str[index] == '}': + if count == 0: + # means we never found an open brace + return (-1, -1) + count -= 1; + + if count == 0 and found: + return (count, index) + + return (count, -1) + + def parse(self, filename): + # Strip the current directory off the start of the + # filename for brevity + if filename[0:len(self.input_prefix)] == self.input_prefix: + filename = filename[len(self.input_prefix):] + if filename[0] == "/": + filename = filename[1:] + + # Work out the abbreviated directory name + directory = os.path.dirname(filename) + if directory[0:2] == "./": + directory = directory[2:] + elif directory[0:len(self.input_prefix)] == self.input_prefix: + directory = directory[len(self.input_prefix):] + if directory[0] == "/": + directory = directory[1:] + + # Open the file and explore its contents... + self.log.info("Siphoning from %s." % filename) + directives = {} + with open(filename) as fd: + siphon = None + close_siphon = None + siphon_block = "" + in_block = False + line_num = 0 + siphon_line = 0 + + for line in fd: + line_num += 1 + str = line[:-1] # filter \n + + """See if there is a block directive and if so extract it""" + def process_block_directive(str, directives): + m = self.siphon_block_directive.search(str) + if m is not None: + k = m.group(2) + v = m.group(3).strip() + directives[k] = v + # Return only the parts we did not match + return str[0:m.start(1)] + str[m.end(4):] + + return str + + def process_block_prefix(str): + if str.startswith(" * "): + str = str[3:] + elif str == " *": + str = "" + return str + + if not in_block: + # See if the line contains the start of a siphon doc block + m = self.siphon_block_start.search(str) + if m is not None: + in_block = True + t = m.group(1) + + # Now check if the block closes on the same line + m = self.siphon_block_stop.search(t) + if m is not None: + t = m.group(1) + in_block = False + + # Check for directives + t = process_block_directive(t, directives) + + # Filter for normal comment prefixes + t = process_block_prefix(t) + + # Add what is left + siphon_block += t + + # Skip to next line + continue + + else: + # Check to see if we have an end block marker + m = self.siphon_block_stop.search(str) + if m is not None: + in_block = False + t = m.group(1) + else: + t = str + + # Check for directives + t = process_block_directive(t, directives) + + # Filter for normal comment prefixes + t = process_block_prefix(t) + + # Add what is left + siphon_block += t + "\n" + + # Skip to next line + continue + + + if siphon is None: + # Look for blocks we need to siphon + for p in siphon_patterns: + if p[0].match(str): + siphon = [ p[1], str + "\n", 0 ] + siphon_line = line_num + + # see if we have an initializer + m = self.siphon_initializer.search(str) + if m is not None: + # count the braces on this line + (count, index) = \ + self.count_braces(str[m.start():]) + siphon[2] = count + # TODO - it's possible we have the + # initializer all on the first line + # we should check for it, but also + # account for the possibility that + # the open brace is on the next line + #if count == 0: + # # braces balanced + # close_siphon = siphon + # siphon = None + else: + # no initializer: close the siphon right now + close_siphon = siphon + siphon = None + else: + # See if we should end the siphon here - do we have + # balanced braces? + (count, index) = self.count_braces(str, + count=siphon[2], found=True) + if count == 0: + # braces balanced - add the substring and + # close the siphon + siphon[1] += str[:index+1] + ";\n" + close_siphon = siphon + siphon = None + else: + # add the whole string, move on + siphon[2] = count + siphon[1] += str + "\n" + + if close_siphon is not None: + # Write the siphoned contents to the right place + siphon_name = close_siphon[0] + + # Copy directives for the file + details = {} + for key in directives: + if ":" in key: + (sn, label) = key.split(":") + if sn == siphon_name: + details[label] = directives[key] + else: + details[key] = directives[key] + + # Copy details for this block + details['file'] = filename + details['directory'] = directory + details['line_start'] = siphon_line + details['line_end'] = line_num + details['siphon_block'] = siphon_block.strip() + details["block"] = close_siphon[1] + + # Store the item + self.output[siphon_name]['items'].append(details) + + # All done + close_siphon = None + siphon_block = "" + + # Update globals + for key in directives.keys(): + if ':' not in key: + continue + + if filename.endswith("/dir.dox"): + # very special! use the parent directory name + l = directory + else: + l = filename + + (sn, label) = key.split(":") + + if sn not in self.output: + self.output[sn] = {} + if 'global' not in self.output[sn]: + self.output[sn]['global'] = {} + if l not in self.output[sn]['global']: + self.output[sn]['global'][l] = {} + + self.output[sn]['global'][l][label] = directives[key] + + def deliver(self): + # Write out the data + for siphon in self.output.keys(): + self.log.info("Saving siphon data %s." % siphon) + s = self.output[siphon] + with open(s['file'], "a") as fp: + json.dump(s, fp, + separators=(',', ': '), indent=4, sort_keys=True) + diff --git a/doxygen/siphon/generate_clicmd.py b/doxygen/siphon/generate_clicmd.py new file mode 100644 index 00000000000..7b13111028d --- /dev/null +++ b/doxygen/siphon/generate_clicmd.py @@ -0,0 +1,22 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import generate, re + +# Register our regexp +generate.siphon_patterns.append(( + re.compile("(?P<m>VLIB_CLI_COMMAND)\s*" + "[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), + "clicmd" +)) diff --git a/doxygen/siphon/generate_syscfg.py b/doxygen/siphon/generate_syscfg.py new file mode 100644 index 00000000000..c77936a9dc8 --- /dev/null +++ b/doxygen/siphon/generate_syscfg.py @@ -0,0 +1,22 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import generate, re + +# Register our regexp +generate.siphon_patterns.append(( + re.compile("(?P<m>VLIB_CONFIG_FUNCTION)\s*" + '[(](?P<fn>[a-zA-Z0-9_]+)\s*,\s*"(?P<name>[^"]*)"[)]'), + "syscfg" +)) diff --git a/doxygen/siphon/parsers.py b/doxygen/siphon/parsers.py new file mode 100644 index 00000000000..6fe8600d4b3 --- /dev/null +++ b/doxygen/siphon/parsers.py @@ -0,0 +1,149 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cgi, pyparsing as pp + +# Some useful primitives +ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_") +intNum = pp.Word(pp.nums) +hexNum = pp.Literal("0x") + pp.Word(pp.hexnums) +octalNum = pp.Literal("0") + pp.Word("01234567") +integer = (hexNum | octalNum | intNum) + \ + pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L")) +floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f")) +char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'") +arrayIndex = integer | ident + +lbracket = pp.Literal("(").suppress() +rbracket = pp.Literal(")").suppress() +lbrace = pp.Literal("{").suppress() +rbrace = pp.Literal("}").suppress() +comma = pp.Literal(",").suppress() +equals = pp.Literal("=").suppress() +dot = pp.Literal(".").suppress() +semicolon = pp.Literal(";").suppress() + +# initializer := { [member = ] (variable | expression | { initializer } ) } +typeName = ident +varName = ident +typeSpec = pp.Optional("unsigned") + \ + pp.oneOf("int long short float double char u8 i8 void") + \ + pp.Optional(pp.Word("*"), default="") +typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress() + +string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"', + escChar='\\', multiline=True)), adjacent=False) +literal = pp.Optional(typeCast) + (integer | floatNum | char | string) +var = pp.Combine(pp.Optional(typeCast) + varName + + pp.Optional("[" + arrayIndex + "]")) + +# This could be more complete, but suffices for our uses +expr = (literal | var) + +"""Parse and render a block of text into a Python dictionary.""" +class Parser(object): + """Compiled PyParsing BNF""" + _parser = None + + def __init__(self): + super(Parser, self).__init__() + self._parser = self.BNF() + + def BNF(self): + raise NotImplementedError + + def item(self, item): + raise NotImplementedError + + def parse(self, input): + item = self._parser.parseString(input).asList() + return self.item(item) + + +"""Parser for function-like macros - without the closing semi-colon.""" +class ParserFunctionMacro(Parser): + def BNF(self): + # VLIB_CONFIG_FUNCTION (unix_config, "unix") + macroName = ident + params = pp.Group(pp.ZeroOrMore(expr + comma) + expr) + macroParams = lbracket + params + rbracket + + return macroName + macroParams + + def item(self, item): + r = { + "macro": item[0], + "name": item[1][1], + "function": item[1][0], + } + + return r + + +"""Parser for function-like macros with a closing semi-colon.""" +class ParseFunctionMacroStmt(ParserFunctionMacro): + def BNF(self): + # VLIB_CONFIG_FUNCTION (unix_config, "unix"); + function_macro = super(ParseFunctionMacroStmt, self).BNF() + mi = function_macro + semicolon + mi.ignore(pp.cppStyleComment) + + return mi + + +""" +Parser for our struct initializers which are composed from a +function-like macro, equals sign, and then a normal C struct initalizer +block. +""" +class MacroInitializer(ParserFunctionMacro): + def BNF(self): + # VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = { + # .path = "show sr tunnel", + # .short_help = "show sr tunnel [name <sr-tunnel-name>]", + # .function = show_sr_tunnel_fn, + # }; + cs = pp.Forward() + + + member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"), + adjacent=False) + value = (expr | cs) + + entry = pp.Group(pp.Optional(member + equals, default="") + value) + entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \ + (pp.ZeroOrMore(entry + comma)) + + cs << (lbrace + entries + rbrace) + + macroName = ident + params = pp.Group(pp.ZeroOrMore(expr + comma) + expr) + macroParams = lbracket + params + rbracket + + function_macro = super(MacroInitializer, self).BNF() + mi = function_macro + equals + pp.Group(cs) + semicolon + mi.ignore(pp.cppStyleComment) + + return mi + + def item(self, item): + r = { + "macro": item[0], + "name": item[1][0], + "params": item[2], + "value": {}, + } + + for param in item[2]: + r["value"][param[0]] = cgi.escape(param[1]) + + return r diff --git a/doxygen/siphon/process.py b/doxygen/siphon/process.py new file mode 100644 index 00000000000..c7f8f1a232b --- /dev/null +++ b/doxygen/siphon/process.py @@ -0,0 +1,271 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generation template class + +import logging, os,sys, cgi, json, jinja2, HTMLParser + +# Classes register themselves in this dictionary +"""Mapping of known processors to their classes""" +siphons = {} + + +"""Generate rendered output for siphoned data.""" +class Siphon(object): + + # Set by subclasses + """Our siphon name""" + name = None + + # Set by subclasses + """Name of an identifier used by this siphon""" + identifier = None + + # Set by subclasses + """The pyparsing object to use to parse with""" + _parser = None + + """The input data""" + _cmds = None + + """Group key to (directory,file) mapping""" + _group = None + + """Logging handler""" + log = None + + """Directory to look for siphon rendering templates""" + template_directory = None + + """Template environment, if we're using templates""" + _tplenv = None + + def __init__(self, template_directory=None): + super(Siphon, self).__init__() + self.log = logging.getLogger("siphon.process.%s" % self.name) + + if template_directory is not None: + self.template_directory = template_directory + searchpath = [ + template_directory + "/" + self.name, + template_directory + "/" + "default", + ] + loader = jinja2.FileSystemLoader(searchpath=searchpath) + self._tplenv = jinja2.Environment( + loader=loader, + trim_blocks=True, + keep_trailing_newline=True) + + # Convenience, get a reference to the internal escape and + # unescape methods in cgi and HTMLParser. These then become + # available to templates to use, if needed. + self._h = HTMLParser.HTMLParser() + self.escape = cgi.escape + self.unescape = self._h.unescape + + + # Output renderers + + """Returns an object to be used as the sorting key in the item index.""" + def index_sort_key(self, group): + return group + + """Returns a string to use as the header at the top of the item index.""" + def index_header(self): + return self.template("index_header") + + """Returns the string fragment to use for each section in the item + index.""" + def index_section(self, group): + return self.template("index_section", group=group) + + """Returns the string fragment to use for each entry in the item index.""" + def index_entry(self, meta, item): + return self.template("index_entry", meta=meta, item=item) + + """Returns an object, typically a string, to be used as the sorting key + for items within a section.""" + def item_sort_key(self, item): + return item['name'] + + """Returns a key for grouping items together.""" + def group_key(self, directory, file, macro, name): + _global = self._cmds['_global'] + + if file in _global and 'group_label' in _global[file]: + self._group[file] = (directory, file) + return file + + self._group[directory] = (directory, None) + return directory + + """Returns a key for identifying items within a grouping.""" + def item_key(self, directory, file, macro, name): + return name + + """Returns a string to use as the header when rendering the item.""" + def item_header(self, group): + return self.template("item_header", group=group) + + """Returns a string to use as the body when rendering the item.""" + def item_format(self, meta, item): + return self.template("item_format", meta=meta, item=item) + + """Returns a string to use as the label for the page reference.""" + def page_label(self, group): + return "_".join(( + self.name, + self.sanitize_label(group) + )) + + """Returns a title to use for a page.""" + def page_title(self, group): + _global = self._cmds['_global'] + (directory, file) = self._group[group] + + if file and file in _global and 'group_label' in _global[file]: + return _global[file]['group_label'] + + if directory in _global and 'group_label' in _global[directory]: + return _global[directory]['group_label'] + + return directory + + """Returns a string to use as the label for the section reference.""" + def item_label(self, group, item): + return "__".join(( + self.name, + item + )) + + """Label sanitizer; for creating Doxygen references""" + def sanitize_label(self, value): + return value.replace(" ", "_") \ + .replace("/", "_") \ + .replace(".", "_") + + """Template processor""" + def template(self, name, **kwargs): + tpl = self._tplenv.get_template(name + ".md") + return tpl.render( + this=self, + **kwargs) + + + # Processing methods + + """Parse the input file into a more usable dictionary structure.""" + def load_json(self, files): + self._cmds = {} + self._group = {} + + line_num = 0 + line_start = 0 + for filename in files: + filename = os.path.relpath(filename) + self.log.info("Parsing items in file \"%s\"." % filename) + data = None + with open(filename, "r") as fd: + data = json.load(fd) + + self._cmds['_global'] = data['global'] + + # iterate the items loaded and regroup it + for item in data["items"]: + try: + o = self._parser.parse(item['block']) + except: + self.log.error("Exception parsing item: %s\n%s" \ + % (json.dumps(item, separators=(',', ': '), + indent=4), + item['block'])) + raise + + # Augment the item with metadata + o["meta"] = {} + for key in item: + if key == 'block': + continue + o['meta'][key] = item[key] + + # Load some interesting fields + directory = item['directory'] + file = item['file'] + macro = o["macro"] + name = o["name"] + + # Generate keys to group items by + group_key = self.group_key(directory, file, macro, name) + item_key = self.item_key(directory, file, macro, name) + + if group_key not in self._cmds: + self._cmds[group_key] = {} + + self._cmds[group_key][item_key] = o + + """Iterate over the input data, calling render methods to generate the + output.""" + def process(self, out=None): + + if out is None: + out = sys.stdout + + # Accumulated body contents + contents = "" + + # Write the header for this siphon type + out.write(self.index_header()) + + # Sort key helper for the index + def group_sort_key(group): + return self.index_sort_key(group) + + # Iterate the dictionary and process it + for group in sorted(self._cmds.keys(), key=group_sort_key): + if group.startswith('_'): + continue + + self.log.info("Processing items in group \"%s\" (%s)." % \ + (group, group_sort_key(group))) + + # Generate the section index entry (write it now) + out.write(self.index_section(group)) + + # Generate the item header (save for later) + contents += self.item_header(group) + + def item_sort_key(key): + return self.item_sort_key(self._cmds[group][key]) + + for key in sorted(self._cmds[group].keys(), key=item_sort_key): + self.log.debug("--- Processing key \"%s\" (%s)." % \ + (key, item_sort_key(key))) + + o = self._cmds[group][key] + meta = { + "directory": o['meta']['directory'], + "file": o['meta']['file'], + "macro": o['macro'], + "key": key, + "label": self.item_label(group, key), + } + + # Generate the index entry for the item (write it now) + out.write(self.index_entry(meta, o)) + + # Generate the item itself (save for later) + contents += self.item_format(meta, o) + + # Deliver the accumulated body output + out.write(contents) diff --git a/doxygen/siphon/process_clicmd.py b/doxygen/siphon/process_clicmd.py new file mode 100644 index 00000000000..9b3bd35c86d --- /dev/null +++ b/doxygen/siphon/process_clicmd.py @@ -0,0 +1,56 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate clicmd formatted output + +import process, parsers + +class SiphonCLICMD(process.Siphon): + + name = "clicmd" + identifier = "VLIB_CLI_COMMAND" + + def __init__(self, *args, **kwargs): + super(SiphonCLICMD, self).__init__(*args, **kwargs) + self._parser = parsers.MacroInitializer() + + + # Output renderers + + def index_sort_key(self, group): + _global = self._cmds['_global'] + if group not in self._group: + return group + (directory, file) = self._group[group] + + if file in _global and 'group_label' in _global[file]: + return _global[file]['group_label'] + + if directory in _global and 'group_label' in _global[directory]: + return _global[directory]['group_label'] + + return group + + def item_sort_key(self, item): + return item['value']['path'] + + def item_label(self, group, item): + return "_".join(( + self.name, + self.sanitize_label(self._cmds[group][item]['value']['path']) + )) + + +# Register our processor +process.siphons["clicmd"] = SiphonCLICMD diff --git a/doxygen/siphon/process_syscfg.py b/doxygen/siphon/process_syscfg.py new file mode 100644 index 00000000000..94be591039d --- /dev/null +++ b/doxygen/siphon/process_syscfg.py @@ -0,0 +1,30 @@ +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate syscfg formatted output + +import process, parsers + +class SiphonSYSCFG(process.Siphon): + + name = "syscfg" + identifier = "VLIB_CONFIG_FUNCTION" + + def __init__(self, *args, **kwargs): + super(SiphonSYSCFG, self).__init__(*args, **kwargs) + self._parser = parsers.ParseFunctionMacroStmt() + + +# Register our processor +process.siphons["syscfg"] = SiphonSYSCFG diff --git a/doxygen/siphon_generate.py b/doxygen/siphon_generate.py deleted file mode 100755 index 8b999114e52..00000000000 --- a/doxygen/siphon_generate.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2016 Comcast Cable Communications Management, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Looks for preprocessor macros with struct initializers and siphons them -# off into another file for later parsing; ostensibly to generate -# documentation from struct initializer data. - -import os, sys, re, argparse, json - -DEFAULT_OUTPUT = "build-root/docs/siphons" -DEFAULT_PREFIX = os.getcwd() - -ap = argparse.ArgumentParser() -ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT, - help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT) -ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX, - help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX) -ap.add_argument("input", nargs='+', metavar="input_file", - help="Input C source files") -args = ap.parse_args() - -"""Patterns that match the start of code blocks we want to siphon""" -siphon_patterns = [ - ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), "clicmd" ), -] - -"""Matches a siphon comment block start""" -siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$") - -"""Matches a siphon comment block stop""" -siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$") - -"""Siphon block directive delimiter""" -siphon_block_delimiter = "%%" - -"""Matches a siphon block directive such as '%clicmd:group_label Debug CLI%'""" -siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \ - (siphon_block_delimiter, siphon_block_delimiter)) - -"""Matches the start of an initializer block""" -siphon_initializer = re.compile("\s*=") - -""" -count open and close braces in str -return (0, index) when braces were found and count becomes 0. -index indicates the position at which the last closing brace was -found. -return (-1, -1) if a closing brace is found before any opening one. -return (count, -1) if not all opening braces are closed, count is the -current depth -""" -def count_braces(str, count=0, found=False): - for index in range(0, len(str)): - if str[index] == '{': - count += 1; - found = True - elif str[index] == '}': - if count == 0: - # means we never found an open brace - return (-1, -1) - count -= 1; - - if count == 0 and found: - return (count, index) - - return (count, -1) - -# Collated output for each siphon -output = {} - -# Build a list of known siphons -known_siphons = [] -for item in siphon_patterns: - siphon = item[1] - if siphon not in known_siphons: - known_siphons.append(siphon) - -# Setup information for siphons we know about -for siphon in known_siphons: - output[siphon] = { - "file": "%s/%s.siphon" % (args.output, siphon), - "global": {}, - "items": [], - } - -# Pre-process file names in case they indicate a file with -# a list of files -files = [] -for filename in args.input: - if filename.startswith('@'): - with open(filename[1:], 'r') as fp: - lines = fp.readlines() - for line in lines: - files.append(line.strip()) - lines = None - else: - files.append(filename) - -# Iterate all the input files we've been given -for filename in files: - # Strip the current directory off the start of the - # filename for brevity - if filename[0:len(args.input_prefix)] == args.input_prefix: - filename = filename[len(args.input_prefix):] - if filename[0] == "/": - filename = filename[1:] - - # Work out the abbreviated directory name - directory = os.path.dirname(filename) - if directory[0:2] == "./": - directory = directory[2:] - elif directory[0:len(args.input_prefix)] == args.input_prefix: - directory = directory[len(args.input_prefix):] - if directory[0] == "/": - directory = directory[1:] - - # Open the file and explore its contents... - sys.stderr.write("Siphoning from %s...\n" % filename) - directives = {} - with open(filename) as fd: - siphon = None - close_siphon = None - siphon_block = "" - in_block = False - line_num = 0 - siphon_line = 0 - - for line in fd: - line_num += 1 - str = line[:-1] # filter \n - - """See if there is a block directive and if so extract it""" - def process_block_directive(str, directives): - m = siphon_block_directive.search(str) - if m is not None: - k = m.group(2) - v = m.group(3).strip() - directives[k] = v - # Return only the parts we did not match - return str[0:m.start(1)] + str[m.end(4):] - - return str - - def process_block_prefix(str): - if str.startswith(" * "): - str = str[3:] - elif str == " *": - str = "" - return str - - if not in_block: - # See if the line contains the start of a siphon doc block - m = siphon_block_start.search(str) - if m is not None: - in_block = True - t = m.group(1) - - # Now check if the block closes on the same line - m = siphon_block_stop.search(t) - if m is not None: - t = m.group(1) - in_block = False - - # Check for directives - t = process_block_directive(t, directives) - - # Filter for normal comment prefixes - t = process_block_prefix(t) - - # Add what is left - siphon_block += t - - # Skip to next line - continue - - else: - # Check to see if we have an end block marker - m = siphon_block_stop.search(str) - if m is not None: - in_block = False - t = m.group(1) - else: - t = str - - # Check for directives - t = process_block_directive(t, directives) - - # Filter for normal comment prefixes - t = process_block_prefix(t) - - # Add what is left - siphon_block += t + "\n" - - # Skip to next line - continue - - - if siphon is None: - # Look for blocks we need to siphon - for p in siphon_patterns: - if p[0].match(str): - siphon = [ p[1], str + "\n", 0 ] - siphon_line = line_num - - # see if we have an initializer - m = siphon_initializer.search(str) - if m is not None: - # count the braces on this line - (count, index) = count_braces(str[m.start():]) - siphon[2] = count - # TODO - it's possible we have the initializer all on the first line - # we should check for it, but also account for the possibility that - # the open brace is on the next line - #if count == 0: - # # braces balanced - # close_siphon = siphon - # siphon = None - else: - # no initializer: close the siphon right now - close_siphon = siphon - siphon = None - else: - # See if we should end the siphon here - do we have balanced - # braces? - (count, index) = count_braces(str, count=siphon[2], found=True) - if count == 0: - # braces balanced - add the substring and close the siphon - siphon[1] += str[:index+1] + ";\n" - close_siphon = siphon - siphon = None - else: - # add the whole string, move on - siphon[2] = count - siphon[1] += str + "\n" - - if close_siphon is not None: - # Write the siphoned contents to the right place - siphon_name = close_siphon[0] - - # Copy directives for the file - details = {} - for key in directives: - if ":" in key: - (sn, label) = key.split(":") - if sn == siphon_name: - details[label] = directives[key] - else: - details[key] = directives[key] - - # Copy details for this block - details['file'] = filename - details['line_start'] = siphon_line - details['line_end'] = line_num - details['siphon_block'] = siphon_block.strip() - - # Some defaults - if "group" not in details: - if "group_label" in details: - # use the filename since group labels are mostly of file scope - details['group'] = details['file'] - else: - details['group'] = directory - - if "group_label" not in details: - details['group_label'] = details['group'] - - details["block"] = close_siphon[1] - - # Store the item - output[siphon_name]['items'].append(details) - - # All done - close_siphon = None - siphon_block = "" - - # Update globals - for key in directives.keys(): - if ':' not in key: - continue - - if filename.endswith("/dir.dox"): - # very special! use the parent directory name - l = directory - else: - l = filename - - (sn, label) = key.split(":") - - if sn not in output: - output[sn] = {} - if 'global' not in output[sn]: - output[sn]['global'] = {} - if l not in output[sn]['global']: - output[sn]['global'][l] = {} - if 'file' not in output[sn]: - output[sn]['file'] = "%s/%s.siphon" % (args.output, sn) - if 'items' not in output[sn]: - output[sn]['items'] = [] - - output[sn]['global'][l][label] = directives[key] - - -# Write out the data -for siphon in output.keys(): - sys.stderr.write("Saving siphon %s...\n" % siphon) - s = output[siphon] - with open(s['file'], "a") as fp: - json.dump(s, fp, separators=(',', ': '), indent=4, sort_keys=True) - -# All done diff --git a/doxygen/siphon_process.py b/doxygen/siphon_process.py deleted file mode 100755 index 82a166d31b5..00000000000 --- a/doxygen/siphon_process.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2016 Comcast Cable Communications Management, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Filter for .siphon files that are generated by other filters. -# The idea is to siphon off certain initializers so that we can better -# auto-document the contents of that initializer. - -import os, sys, re, argparse, cgi, json -import pyparsing as pp - -import pprint - -DEFAULT_SIPHON ="clicmd" -DEFAULT_OUTPUT = None -DEFAULT_PREFIX = os.getcwd() - -siphon_map = { - 'clicmd': "VLIB_CLI_COMMAND", -} - -ap = argparse.ArgumentParser() -ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON, - choices=siphon_map.keys(), - help="Siphon type to process [%s]" % DEFAULT_SIPHON) -ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT, - help="Output directory for .md files [%s]" % DEFAULT_OUTPUT) -ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX, - help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX) -ap.add_argument("input", nargs='+', metavar="input_file", - help="Input .siphon files") -args = ap.parse_args() - -if args.output is None: - sys.stderr.write("Error: Siphon processor requires --output to be set.") - sys.exit(1) - - -def clicmd_index_sort(cfg, group, dec): - if group in dec and 'group_label' in dec[group]: - return dec[group]['group_label'] - return group - -def clicmd_index_header(cfg): - s = "# CLI command index\n" - s += "\n[TOC]\n" - return s - -def clicmd_index_section(cfg, group, md): - return "\n@subpage %s\n\n" % md - -def clicmd_index_entry(cfg, meta, item): - v = item["value"] - return "* [%s](@ref %s)\n" % (v["path"], meta["label"]) - -def clicmd_sort(cfg, meta, item): - return item['value']['path'] - -def clicmd_header(cfg, group, md, dec): - if group in dec and 'group_label' in dec[group]: - label = dec[group]['group_label'] - else: - label = group - return "\n@page %s %s\n" % (md, label) - -def clicmd_format(cfg, meta, item): - v = item["value"] - s = "\n@section %s %s\n" % (meta['label'], v['path']) - - # The text from '.short_help = '. - # Later we should split this into short_help and usage_help - # since the latter is how it is primarily used but the former - # is also needed. - if "short_help" in v: - tmp = v["short_help"].strip() - - # Bit hacky. Add a trailing period if it doesn't have one. - if tmp[-1] != ".": - tmp += "." - - s += "### Summary/usage\n %s\n\n" % tmp - - # This is seldom used and will likely be deprecated - if "long_help" in v: - tmp = v["long_help"] - - s += "### Long help\n %s\n\n" % tmp - - # Extracted from the code in /*? ... ?*/ blocks - if "siphon_block" in item["meta"]: - sb = item["meta"]["siphon_block"] - - if sb != "": - # hack. still needed? - sb = sb.replace("\n", "\\n") - try: - sb = json.loads('"'+sb+'"') - s += "### Description\n%s\n\n" % sb - except: - pass - - # Gives some developer-useful linking - if "item" in meta or "function" in v: - s += "### Declaration and implementation\n\n" - - if "item" in meta: - s += "Declaration: @ref %s (%s:%d)\n\n" % \ - (meta['item'], meta["file"], int(item["meta"]["line_start"])) - - if "function" in v: - s += "Implementation: @ref %s.\n\n" % v["function"] - - return s - - -siphons = { - "VLIB_CLI_COMMAND": { - "index_sort_key": clicmd_index_sort, - "index_header": clicmd_index_header, - "index_section": clicmd_index_section, - "index_entry": clicmd_index_entry, - 'sort_key': clicmd_sort, - "header": clicmd_header, - "format": clicmd_format, - } -} - - -# PyParsing definition for our struct initializers which look like this: -# VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = { -# .path = "show sr tunnel", -# .short_help = "show sr tunnel [name <sr-tunnel-name>]", -# .function = show_sr_tunnel_fn, -#}; -def getMacroInitializerBNF(): - cs = pp.Forward() - ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_") - intNum = pp.Word(pp.nums) - hexNum = pp.Literal("0x") + pp.Word(pp.hexnums) - octalNum = pp.Literal("0") + pp.Word("01234567") - integer = (hexNum | octalNum | intNum) + \ - pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L")) - floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f")) - char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'") - arrayIndex = integer | ident - - lbracket = pp.Literal("(").suppress() - rbracket = pp.Literal(")").suppress() - lbrace = pp.Literal("{").suppress() - rbrace = pp.Literal("}").suppress() - comma = pp.Literal(",").suppress() - equals = pp.Literal("=").suppress() - dot = pp.Literal(".").suppress() - semicolon = pp.Literal(";").suppress() - - # initializer := { [member = ] (variable | expression | { initializer } ) } - typeName = ident - varName = ident - - typeSpec = pp.Optional("unsigned") + \ - pp.oneOf("int long short float double char u8 i8 void") + \ - pp.Optional(pp.Word("*"), default="") - typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress() - - string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"', - escChar='\\', multiline=True)), adjacent=False) - literal = pp.Optional(typeCast) + (integer | floatNum | char | string) - var = pp.Combine(pp.Optional(typeCast) + varName + pp.Optional("[" + arrayIndex + "]")) - - expr = (literal | var) # TODO - - - member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"), adjacent=False) - value = (expr | cs) - - entry = pp.Group(pp.Optional(member + equals, default="") + value) - entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \ - (pp.ZeroOrMore(entry + comma)) - - cs << (lbrace + entries + rbrace) - - macroName = ident - params = pp.Group(pp.ZeroOrMore(expr + comma) + expr) - macroParams = lbracket + params + rbracket - - mi = macroName + pp.Optional(macroParams) + equals + pp.Group(cs) + semicolon - mi.ignore(pp.cppStyleComment) - return mi - - -mi = getMacroInitializerBNF() - -# Parse the input file into a more usable dictionary structure -cmds = {} -line_num = 0 -line_start = 0 -for filename in args.input: - sys.stderr.write("Parsing items in file \"%s\"...\n" % filename) - data = None - with open(filename, "r") as fd: - data = json.load(fd) - - cmds['_global'] = data['global'] - - # iterate the items loaded and regroup it - for item in data["items"]: - try: - o = mi.parseString(item['block']).asList() - except: - sys.stderr.write("Exception parsing item: %s\n%s\n" \ - % (json.dumps(item, separators=(',', ': '), indent=4), - item['block'])) - raise - - group = item['group'] - file = item['file'] - macro = o[0] - param = o[1][0] - - if group not in cmds: - cmds[group] = {} - - if file not in cmds[group]: - cmds[group][file] = {} - - if macro not in cmds[group][file]: - cmds[group][file][macro] = {} - - c = { - 'params': o[2], - 'meta': {}, - 'value': {}, - } - - for key in item: - if key == 'block': - continue - c['meta'][key] = item[key] - - for i in c['params']: - c['value'][i[0]] = cgi.escape(i[1]) - - cmds[group][file][macro][param] = c - - -# Write the header for this siphon type -cfg = siphons[siphon_map[args.type]] -sys.stdout.write(cfg["index_header"](cfg)) -contents = "" - -def group_sort_key(item): - if "index_sort_key" in cfg: - return cfg["index_sort_key"](cfg, item, cmds['_global']) - return item - -# Iterate the dictionary and process it -for group in sorted(cmds.keys(), key=group_sort_key): - if group.startswith('_'): - continue - - sys.stderr.write("Processing items in group \"%s\"...\n" % group) - - cfg = siphons[siphon_map[args.type]] - md = group.replace("/", "_").replace(".", "_") - sys.stdout.write(cfg["index_section"](cfg, group, md)) - - if "header" in cfg: - dec = cmds['_global'] - contents += cfg["header"](cfg, group, md, dec) - - for file in sorted(cmds[group].keys()): - if group.startswith('_'): - continue - - sys.stderr.write("- Processing items in file \"%s\"...\n" % file) - - for macro in sorted(cmds[group][file].keys()): - if macro != siphon_map[args.type]: - continue - sys.stderr.write("-- Processing items in macro \"%s\"...\n" % macro) - cfg = siphons[macro] - - meta = { - "group": group, - "file": file, - "macro": macro, - "md": md, - } - - def item_sort_key(item): - if "sort_key" in cfg: - return cfg["sort_key"](cfg, meta, cmds[group][file][macro][item]) - return item - - for param in sorted(cmds[group][file][macro].keys(), key=item_sort_key): - sys.stderr.write("--- Processing item \"%s\"...\n" % param) - - meta["item"] = param - - # mangle "md" and the item to make a reference label - meta["label"] = "%s___%s" % (meta["md"], param) - - if "index_entry" in cfg: - s = cfg["index_entry"](cfg, meta, cmds[group][file][macro][param]) - sys.stdout.write(s) - - if "format" in cfg: - contents += cfg["format"](cfg, meta, cmds[group][file][macro][param]) - -sys.stdout.write(contents) - -# All done diff --git a/doxygen/siphon_templates/clicmd/index_entry.md b/doxygen/siphon_templates/clicmd/index_entry.md new file mode 100644 index 00000000000..1fa9ec9b25a --- /dev/null +++ b/doxygen/siphon_templates/clicmd/index_entry.md @@ -0,0 +1,17 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% set v = item['value'] %} +{{ "* [%s](@ref %s)" % (v['path'], meta["label"]) }} diff --git a/doxygen/siphon_templates/clicmd/index_header.md b/doxygen/siphon_templates/clicmd/index_header.md new file mode 100644 index 00000000000..4167f4dc477 --- /dev/null +++ b/doxygen/siphon_templates/clicmd/index_header.md @@ -0,0 +1,130 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +# Debug CLI {{'{#'}}clicmd} + +The VPP network stack comes equipped with a set of commands that are useful +for debugging. + +The easiest way to access the CLI (with proper permissions) is to use the +vppctl command: + +``` +sudo vppctl <cli-command> +``` + +The CLI parser matches static keyword strings, eventually invoking an action +function. Unambiguous partial keyword matching always occurs. The action +functions consume input until satisfied or until they fail. This model makes +for easy coding, but does not guarantee useful "help" output. It's up to the +CLI command writer to add useful help strings. + +You can find the source code of CLI commands by searching for instances of the +@c VLIB_CLI_COMMAND macro in the code source files. + +Please help maintain and improve this document to make and keep these commands +clear and useful! + +@todo Document where to modify this CLI intro text. + + +## Debug and Telnet CLI + +The debug CLI is enabled with the unix interactive parameter or startup +configuration option. This causes VPP to start without daemonizing and +presents a command line interface on the terminal where it is run. + +The Telnet CLI is enabled with the `cli-listen localhost:5002` option which +will cause VPP to listen for TCP connections on the localhost address port +@c 5002. A Telnet client can then connect to this port (for example, `telnet +localhost 5002`) and will receive a command line prompt. + +This configuration will enable both mechanisms: + +``` +unix { + interactive + cli-listen localhost:5002 +} +``` + +The debug CLI can operate in line mode, which may be useful when running +inside an IDE like Emacs. This is enabled with the option +`unix cli-line-mode`. Several other options exist that alter how this +CLI works, see the @ref syscfg section for details. + +The CLI starts with a banner graphic (which can be disabled) and a prompt. The +prompt will typically read `vpp` for a release version of VPP and `DBGvpp#` +for a development version with debugging enabled, for example: + + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + vpp# + +versus: + + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + DBGvpp# + +This prompt can be configured with the `unix cli-prompt` setting and the +banner is disabled with `unix cli-no-banner`. + +## CLI features + +The CLI has several editing features that make it easy to use. + +- Cursor keys left/right will move the cursor within a command line; + typing will insert at the cursor; erase will erase at the cursor. + +- Ctrl-left/right will search for the start of the next word to + the left or right. +- Home/end will jump the cursor to the start and end of the line. +- Cursor keys up/down and ^P/^N iterate through the command history + buffer. Lines from the history buffer may be edited. New commands + are added to the end of the buffer when executed; though + duplicates of the previous command are not added. +- ^U erases the line contents from the left of the cursor to the + start. +- ^K erases the contents from the cursor to the end. +- ^S/^R will search the command history forwards or in reverse for + a command; start typing for matches to auto complete. +- ^L will clear the screen (if supported by the terminal) and repaint + the prompt and any current line. The cursor position is also + retained. +- The CLI can be closed with the quit command. Alternatively, ^D on + an empty input line will also close the session. Closing the debug + session will also shutdown VPP. + +Output that exceeds the length of a terminal page will be buffered, up to a +limit. + +- Space or page-down displays the next page. +- Enter or down-arrow displays the next line. +- Page-up goes back a page. +- Up-arrow goes up a line. +- Home/end jump to the start/end of the buffered output. +- The key q quits the pager. Space and enter will also quit the + pager if the end of the buffer has been reached. + +## Index of CLI commands + +[TOC] diff --git a/doxygen/siphon_templates/clicmd/item_format.md b/doxygen/siphon_templates/clicmd/item_format.md new file mode 100644 index 00000000000..77d0484bfb7 --- /dev/null +++ b/doxygen/siphon_templates/clicmd/item_format.md @@ -0,0 +1,59 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% set v = item['value'] %} +{{ "@section %s %s" % (meta['label'], v['path']) }} +{% if 'short_help' in v %} + +### Summary/usage + +{% set str = v['short_help'] %} +{% set period = "." if str[-1] != "." else "" %} +{% set prefix = " " if "[" in str or "<" in str or "|" in str else "" %} +{% set str = this.unescape(str) %} +{{ "%s%s%s" % (prefix, str, period) }} +{% endif %} +{% if 'long_help' in v %} +{# This is seldom used and will likely be deprecated #} + +### Long help + +{{ v['long_help'] }} +{%- endif %} +{% if 'siphon_block' in item['meta'] %} +{% set sb = item["meta"]["siphon_block"] %} +{% if sb %} +{# Extracted from the code in /*? ... ?*/ blocks #} + +### Description + +{{ sb }} +{% endif %} +{% endif %} +{% if "item" in meta or "function" in v %} +{# Gives some developer-useful linking #} + +### Declaration and implementation +{% if "item" in meta %} + +{{ "Declaration: @ref %s (@ref %s line %d)" % + (meta['item'], meta["file"], item["meta"]["line_start"]) }} +{% endif %} +{% if "function" in v %} + +{{ "Implementation: @ref %s." % v["function"] }} +{% endif %} +{% endif %} + diff --git a/doxygen/siphon_templates/default/index_entry.md b/doxygen/siphon_templates/default/index_entry.md new file mode 100644 index 00000000000..479dcdb2cc3 --- /dev/null +++ b/doxygen/siphon_templates/default/index_entry.md @@ -0,0 +1,16 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{{ "* [%s](@ref %s)" % (item["name"], meta["label"]) }} diff --git a/doxygen/siphon_templates/default/index_section.md b/doxygen/siphon_templates/default/index_section.md new file mode 100644 index 00000000000..3c9d2b474c5 --- /dev/null +++ b/doxygen/siphon_templates/default/index_section.md @@ -0,0 +1,18 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +@subpage {{ this.page_label(group) }} + diff --git a/doxygen/siphon_templates/default/item_format.md b/doxygen/siphon_templates/default/item_format.md new file mode 100644 index 00000000000..ed1b1bf7eeb --- /dev/null +++ b/doxygen/siphon_templates/default/item_format.md @@ -0,0 +1,16 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{{ raise NotImplementedError }} diff --git a/doxygen/siphon_templates/default/item_header.md b/doxygen/siphon_templates/default/item_header.md new file mode 100644 index 00000000000..0c21e51fa5d --- /dev/null +++ b/doxygen/siphon_templates/default/item_header.md @@ -0,0 +1,18 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{{ "@page %s %s" % (this.page_label(group), this.page_title(group)) }} + diff --git a/doxygen/siphon_templates/syscfg/index_header.md b/doxygen/siphon_templates/syscfg/index_header.md new file mode 100644 index 00000000000..5d338a0472b --- /dev/null +++ b/doxygen/siphon_templates/syscfg/index_header.md @@ -0,0 +1,111 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +# Startup Configuration {{'{#'}}syscfg} + +The VPP network stack comes with several configuration options that can be +provided either on the command line or in a configuration file. + +Specific applications built on the stack have been known to require a dozen +arguments, depending on requirements. This section describes commonly-used +options and parameters. + +You can find command-line argument parsers in the source code by searching for +instances of the `VLIB_CONFIG_FUNCTION` macro. The invocation +`VLIB_CONFIG_FUNCTION(foo_config, "foo")` will cause the function +`foo_config` to receive all the options and values supplied in a parameter +block named "`foo`", for example: `foo { arg1 arg2 arg3 ... }`. + +@todo Tell the nice people where this document lives so that the might +help improve it! + +## Command-line arguments + +Parameters are grouped by a section name. When providing more than one +parameter to a section all parameters for that section must be wrapped in +curly braces. + +``` +/usr/bin/vpp unix { interactive cli-listen 127.0.0.1:5002 } +``` + +Which will produce output similar to this: + + <startup diagnostic messages> + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + vpp# <start-typing> + +When providing only one such parameter the braces are optional. For example, +the following command argument, `unix interactive` does not have braces: + +``` +/usr/bin/vpp unix interactive +``` + +The command line can be presented as a single string or as several; anything +given on the command line is concatenated with spaces into a single string +before parsing. + +VPP applications must be able to locate their own executable images. The +simplest way to ensure this will work is to invoke a VPP application by giving +its absolute path; for example: `/usr/bin/vpp <options>`. At startup, VPP +applications parse through their own ELF-sections (primarily) to make lists +of init, configuration, and exit handlers. + +When developing with VPP, in _gdb_ it's often sufficient to start an application +like this at the `(gdb)` prompt: + +``` +run unix interactive +``` + +## Configuration file + +It is also possible to supply parameters in a startup configuration file the +path of which is provided to the VPP application on its command line. + +The format of the configuration file is a simple text file with the same +content as the command line but with the benefit of being able to use newlines +to make the content easier to read. For example: + +``` +unix { + nodaemon + log /tmp/vpp.log + full-coredump + cli-listen localhost:5002 +} +api-trace { + on +} +dpdk { + dev 0000:03:00.0 +} +``` + +VPP is then instructed to load this file with the `-c` option: + +``` +/usr/bin/vpp -c /etc/vpp/startup.conf +``` + +## Index of startup command sections + +[TOC] + diff --git a/doxygen/siphon_templates/syscfg/item_format.md b/doxygen/siphon_templates/syscfg/item_format.md new file mode 100644 index 00000000000..5d3bb5c83fb --- /dev/null +++ b/doxygen/siphon_templates/syscfg/item_format.md @@ -0,0 +1,42 @@ +{# +# Copyright (c) 2016 Comcast Cable Communications Management, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +{% set v = item['value'] %} +{{ "@section %s %s" % (meta['label'], item['name']) }} +{% if 'siphon_block' in item['meta'] %} +{% set sb = item["meta"]["siphon_block"] %} +{% if sb %} +{# Extracted from the code in /*? ... ?*/ blocks #} + +### Description + +{{ sb }} +{% endif %} +{% endif %} +{% if "item" in meta or "function" in item %} +{# Gives some developer-useful linking #} + +### Declaration and implementation +{% if "item" in meta %} + +{{ "Declaration: @ref %s (@ref %s line %d)" % + (meta['item'], meta["file"], item["meta"]["line_start"]) }} +{% endif %} +{% if "function" in item %} + +{{ "Implementation: @ref %s." % item["function"] }} +{% endif %} +{% endif %} + diff --git a/vlib/vlib/unix/cj.c b/vlib/vlib/unix/cj.c index 553b450ffaa..33ba163abca 100644 --- a/vlib/vlib/unix/cj.c +++ b/vlib/vlib/unix/cj.c @@ -17,6 +17,16 @@ *------------------------------------------------------------------ */ +/** + * @file + * Circular joournal diagnostic mechanism. + * + * The @c cj thread-safe circular log buffer scheme is occasionally useful + * when chasing bugs. Calls to it should not be checked in. + */ +/*? %%clicmd:group_label Circular Journal %% ?*/ +/*? %%syscfg:group_label Circular Journal %% ?*/ + #include <stdio.h> #include <vlib/vlib.h> @@ -94,6 +104,18 @@ cj_config (vlib_main_t * vm, unformat_input_t * input) return 0; } +/*? + * Configure the circular journal diagnostic mechanism. This is only useful + * if you, the deveoper, have written code to make use of the circular + * journal. + * + * @cfgcmd{records, <number>} + * Configure the number of records to allocate for the circular journal. + * + * @cfgcmd{on} + * Enable the collection of records in the circular journal at the + * earliest opportunity. +?*/ VLIB_CONFIG_FUNCTION (cj_config, "cj"); void @@ -220,10 +242,21 @@ cj_command_fn (vlib_main_t * vm, return 0; } +/*? + * Enable, disable the collection of diagnostic data into a + * circular journal or dump the circular journal diagnostic data. + * This is only useful if you, the deveoper, have written code to make + * use of the circular journal. + * + * When dumping the data it is formatted and sent to @c stderr of the + * VPP process; when running VPP in <code>unix interactive</code> mode + * this is typically the same place as the Debug CLI. +?*/ + /* *INDENT-OFF* */ VLIB_CLI_COMMAND (cj_command,static) = { .path = "cj", - .short_help = "cj", + .short_help = "cj <enable | disable | dump>", .function = cj_command_fn, }; /* *INDENT-ON* */ diff --git a/vlib/vlib/unix/cli.c b/vlib/vlib/unix/cli.c index bf09ee0cce2..69fca6ec7bc 100644 --- a/vlib/vlib/unix/cli.c +++ b/vlib/vlib/unix/cli.c @@ -42,7 +42,8 @@ * Provides a command line interface so humans can interact with VPP. * This is predominantly a debugging and testing mechanism. */ -/*? %%clicmd:group_label Debug CLI %% ?*/ +/*? %%clicmd:group_label Command line session %% ?*/ +/*? %%syscfg:group_label Command line session %% ?*/ #include <vlib/vlib.h> #include <vlib/unix/unix.h> @@ -2504,6 +2505,9 @@ unix_cli_config (vlib_main_t * vm, unformat_input_t * input) return 0; } +/*? + * This module has no configurable parameters. +?*/ VLIB_CONFIG_FUNCTION (unix_cli_config, "unix-cli"); /** Called when VPP is shutting down, this restores the system diff --git a/vlib/vlib/unix/dir.dox b/vlib/vlib/unix/dir.dox index cdded0f19d3..1380fa56b37 100644 --- a/vlib/vlib/unix/dir.dox +++ b/vlib/vlib/unix/dir.dox @@ -23,5 +23,6 @@ VLIB application library Unix interface layer. */ -/*? %%clicmd:group_label VLIB Unix stuff%% ?*/ +/*? %%clicmd:group_label Unix Interface %% ?*/ +/*? %%syscfg:group_label Unix Interface %% ?*/ diff --git a/vlib/vlib/unix/main.c b/vlib/vlib/unix/main.c index 3087c283d9e..562778e0e5d 100644 --- a/vlib/vlib/unix/main.c +++ b/vlib/vlib/unix/main.c @@ -407,6 +407,59 @@ unix_config (vlib_main_t * vm, unformat_input_t * input) } /* unix { ... } configuration. */ +/*? + * + * @cfgcmd{interactive} + * Attach CLI to stdin/out and provide a debugging command line interface. + * Implies @c nodaemon. + * + * @cfgcmd{nodaemon} + * Do not fork or background the VPP process. Typically used when invoking + * VPP applications from a process monitor. + * + * @cfgcmd{exec, <filename>} + * @par <code>startup-config <filename></code> + * Read startup operational configuration from @c filename. + * The contents of the file will be performed as though entered at the CLI. + * The two keywords are aliases for the same function; if both are specified, + * only the last will have an effect. + * + * @cfgcmd{log, <filename>} + * Logs the startup configuration and all subsequent CLI commands in + * @c filename. + * Very useful in situations where folks don't remember or can't be bothered + * to include CLI commands in bug reports. + * + * @cfgcmd{full-coredump} + * Ask the Linux kernel to dump all memory-mapped address regions, instead + * of just text+data+bss. + * + * @cfgcmd{cli-listen, <address:port>} + * Bind the CLI to listen at the address and port given. @clocalhost + * on TCP port @c 5002, given as <tt>cli-listen localhost:5002</tt>, + * is typical. + * + * @cfgcmd{cli-line-mode} + * Disable character-by-character I/O on stdin. Useful when combined with, + * for example, <tt>emacs M-x gud-gdb</tt>. + * + * @cfgcmd{cli-prompt, <string>} + * Configure the CLI prompt to be @c string. + * + * @cfgcmd{cli-history-limit, <nn>} + * Limit commmand history to @c nn lines. A value of @c 0 + * disables command history. Default value: @c 50 + * + * @cfgcmd{cli-no-banner} + * Disable the login banner on stdin and Telnet connections. + * + * @cfgcmd{cli-no-pager} + * Disable the output pager. + * + * @cfgcmd{cli-pager-buffer-limit, <nn>} + * Limit pager buffer to @c nn lines of output. + * A value of @c 0 disables the pager. Default value: @c 100000 +?*/ VLIB_CONFIG_FUNCTION (unix_config, "unix"); static clib_error_t * |