docs: make docs build incremental

This patch makes the `make docs` directive incremental
avoiding re-running the siphon when the source hasn't
changed, and leveraging sphinx internal cache.
It adds a `make rebuild-docs` directive for cases where
this caching logic might break, e.g. in CI.
The virtualenv doesn't also get recreated on each build,
which might be enough when writing docs, provided
automated process leverage its rebuild counterpart.

Type: improvement

Change-Id: Ie90de3adebeed017b249cad81c6c160719f71e8d
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

5 files changed, 69 insertions, 51 deletions

diff --git a/Makefile b/Makefile
index 609a17418fc..6f5bc9be856 100644
--- a/Makefile
+++ b/Makefile
@@ -247,6 +247,7 @@ help:
 	@echo " docs                 - Build the Sphinx documentation"
 	@echo " docs-venv            - Build the virtual environment for the Sphinx docs"
 	@echo " docs-clean           - Remove the generated files from the Sphinx docs"
+	@echo " docs-rebuild         - Rebuild all of the Sphinx documentation"
 	@echo ""
 	@echo "Make Arguments:"
 	@echo " V=[0|1]                  - set build verbosity level"
diff --git a/docs/Makefile b/docs/Makefile
index 71ee034b331..01e8d659bff 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -21,7 +21,6 @@ WS_ROOT ?= $(CURDIR)/..
 BR ?= $(WS_ROOT)/build-root
 DOCS_DIR ?= $(WS_ROOT)/docs
 
-VENV_DIR ?= $(DOCS_DIR)/venv
 SPHINX_SCRIPTS_DIR ?= $(WS_ROOT)/docs/scripts
 
 # Work out the OS if we haven't already
@@ -40,11 +39,11 @@ endif
 SPHINXOPTS        = --keep-going -n -W
 SPHINXBUILD       = sphinx-build
 SPHINXPROJ        = fdio-vpp
-SOURCEDIR         = .
 BUILDDIR          = ${BR}/docs
 BUILDDIR_SRC      = ${BUILDDIR}/src
 BUILDDIR_OUT      = ${BUILDDIR}/html
 SCRIPTS_DIR       = _scripts
+VENV_DIR          ?= $(BUILDDIR)/venv
 
 
 # Put it first so that "make" without argument is like "make help".
@@ -55,28 +54,6 @@ help:
 	  $(SPHINXBUILD) --help ;\
 	)
 
-.PHONY: checkdeps
-checkdeps:
-	@echo "Checking whether dependencies for Docs are installed..."
-ifeq ($(OS_ID),ubuntu)
-	@set -e; inst=; \
-		for i in $(DOC_DEB_DEPENDS); do \
-			dpkg-query --show $$i >/dev/null 2>&1 || inst="$$inst $$i"; \
-		done; \
-		if [ "$$inst" ]; then \
-			sudo apt-get update; \
-			sudo apt-get $(CONFIRM) $(FORCE) install $$inst; \
-		fi
-endif
-
-.PHONY: spell
-spell: clean checkdeps venv ${BUILDDIR_SRC}
-	@( \
-	  . ${VENV_DIR}/bin/activate; \
-	  make -C ${SCRIPTS_DIR} generate && \
-	  $(SPHINXBUILD) -b spelling $(SPHINXOPTS) $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
-	)
-
 .PHONY: venv
 venv:
 	@( \
@@ -89,22 +66,30 @@ venv:
 	    fi; \
 	)
 
-${BUILDDIR_SRC}:
-	@mkdir -p ${BUILDDIR_SRC}
-	@cp -r $(SOURCEDIR) ${BUILDDIR_SRC}
-	@cd ${BUILDDIR_SRC} && find . -type l -exec cp --remove-destination -L ${DOCS_DIR}/{} {} \;
+.PHONY: spell
+spell: venv
+	@( \
+	  . ${VENV_DIR}/bin/activate; \
+	  make -C ${SCRIPTS_DIR} generate && \
+	  $(SPHINXBUILD) -b spelling $(SPHINXOPTS) $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
+	)
+
+.PHONY: rebuild-spell
+rebuild-spell: clean spell
 
 .PHONY: docs
-docs: clean venv ${BUILDDIR_SRC}
+docs: venv
 	@( \
 	  . ${VENV_DIR}/bin/activate; \
 	  make -C ${SCRIPTS_DIR} generate && \
 	  $(SPHINXBUILD) $(SPHINXOPTS) -b html $(BUILDDIR_SRC) $(BUILDDIR_OUT); \
 	)
 
+.PHONY: rebuild
+rebuild: clean docs
+
 .PHONY: clean
 clean:
-	@rm -rf $(BUILDDIR) ${VENV_DIR}
 	@make -C ${SCRIPTS_DIR} clean
 
 .PHONY: build
diff --git a/docs/_scripts/Makefile b/docs/_scripts/Makefile
index f9cb535182f..7893f92cf72 100644
--- a/docs/_scripts/Makefile
+++ b/docs/_scripts/Makefile
@@ -21,8 +21,11 @@
 all: siphon
 
 # These should be passed in by the root Makefile
-WS_ROOT ?= $(CURDIR)/../..
-BR ?= $(WS_ROOT)/build-root
+WS_ROOT           ?= $(CURDIR)/../..
+BR                ?= $(WS_ROOT)/build-root
+BUILDDIR          ?= ${BR}/docs
+BUILDDIR_SRC      ?= ${BUILDDIR}/src
+DOCS_DIR          ?= ${WS_ROOT}/docs
 
 # Tag used in github repository path.
 # Change this when genearting for a release
@@ -34,12 +37,11 @@ REPOSITORY_URL ?= https://github.com/FDio/vpp/blob/$(VPP_TAG)/
 SCRIPTS_DIR ?= $(WS_ROOT)/docs/_scripts
 
 # docs root directory
-DOCS_DIR ?= ${BR}/docs/src
 
-FEATURE_LIST_FILE = ${DOCS_DIR}/aboutvpp/featurelist.md
+FEATURE_LIST_FILE = ${BUILDDIR_SRC}/aboutvpp/featurelist.md
 
 # Siphoned fragements are processed into here
-DOCS_GENERATED_DIR ?= $(DOCS_DIR)/_generated
+DOCS_GENERATED_DIR ?= $(BUILDDIR_SRC)/_generated
 
 # Siphoned fragments end up in here
 SIPHON_INPUT_DIR ?= $(DOCS_GENERATED_DIR)/fragments
@@ -118,8 +120,7 @@ SIPHON_DOCS = $(addprefix $(DOCS_GENERATED_DIR)/,$(addsuffix .rst,$(SIPHONS)))
 BUILT_ON = $(shell date '+%d %B %Y')
 VPP_VERSION = $(shell ${WS_ROOT}/src/scripts/version)
 
-.PHONY: featurelist
-featurelist:
+$(DOCS_GENERATED_DIR)/.featurelist.done:
 	@( \
 	  cd $(WS_ROOT) && \
 	  find . -name FEATURE.yaml | \
@@ -128,19 +129,19 @@ featurelist:
 	      --repolink $(REPOSITORY_URL) > \
 	    $(FEATURE_LIST_FILE) ; \
 	)
+	@touch $(DOCS_GENERATED_DIR)/.featurelist.done
 
 
-.PHONY: includes-render
-includes-render:
+$(DOCS_GENERATED_DIR)/.includes-render.done:
 	@mkdir -p "$(DYNAMIC_RENDER_DIR)"
 	@python3 $(SCRIPTS_DIR)/includes_renderer.py ${WS_ROOT} ${DYNAMIC_RENDER_DIR}
+	@touch $(DOCS_GENERATED_DIR)/.includes-render.done
 
-.PHONY: template-index
-template-index:
-	@sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${DOCS_DIR}/index.rst
-	@sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${DOCS_DIR}/index.rst
+$(DOCS_GENERATED_DIR)/.template-index.done:
+	@sed -ie "s/__VPP_VERSION__/${VPP_VERSION}/g" ${BUILDDIR_SRC}/index.rst
+	@sed -ie "s/__BUILT_ON__/${BUILT_ON}/g" ${BUILDDIR_SRC}/index.rst
 	@( \
-	  for f in $$(grep -l -R __REPOSITORY_URL__ ${DOCS_DIR} | grep -e '\.rst$$' -e '\.md$$' ) ;\
+	  for f in $$(grep -l -R __REPOSITORY_URL__ ${BUILDDIR_SRC} | grep -e '\.rst$$' -e '\.md$$' ) ;\
 	  do \
 	    if [ ! -z $${f} ]; then \
 		echo "TEMPLATING $${f}" ;\
@@ -148,6 +149,7 @@ template-index:
 	    fi ;\
 	  done ; \
 	)
+	@touch $(DOCS_GENERATED_DIR)/.template-index.done
 
 .NOTPARALLEL: $(SIPHON_FILES)
 $(SIPHON_FILES): $(SCRIPTS_DIR)/siphon-generate \
@@ -207,15 +209,45 @@ $(eval $(call siphon-process,rst,markdown))
 
 # This target can be used just to generate the siphoned things
 .PHONY: siphon
-siphon: $(SIPHON_DOCS)
-	@cp $(DOCS_GENERATED_DIR)/clicmd.rst $(DOCS_DIR)/cli-reference/index.rst
-	@cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir $(DOCS_DIR)/cli-reference/clis
+$(DOCS_GENERATED_DIR)/.siphon.done: $(SIPHON_DOCS)
+	@cp $(DOCS_GENERATED_DIR)/clicmd.rst $(BUILDDIR_SRC)/cli-reference/index.rst
+	@mkdir -p $(BUILDDIR_SRC)/cli-reference/clis
+	@cp -r $(DOCS_GENERATED_DIR)/clicmd.rst.dir/* $(BUILDDIR_SRC)/cli-reference/clis
+	@touch $(DOCS_GENERATED_DIR)/.siphon.done
+
+.PHONY: clean-siphons
+clean-siphons:
+	@( \
+	    echo "find $(SIPHON_INPUT) -newer $(DOCS_GENERATED_DIR)/.siphon.done" ; \
+	    cd $(WS_ROOT); \
+	    if [ -f $(DOCS_GENERATED_DIR)/.siphon.done ] && \
+	       [ $$(find $(SIPHON_INPUT) -type f -newer $(DOCS_GENERATED_DIR)/.siphon.done \
+	        			 -not -name '*.md' \
+	        			 -not -name '*.rst' | wc -l) -gt 0  ]; then \
+	      rm -r $(DOCS_GENERATED_DIR); \
+	      echo "removing... $(DOCS_GENERATED_DIR)"; \
+	    fi; \
+	)
+
+${BUILDDIR}/.docsrc.sync.ok:
+	@echo "Copying docs files..."
+	@cp -r $(DOCS_DIR) ${BUILDDIR_SRC}
+	@cd ${BUILDDIR_SRC} && find . -type l -exec cp --remove-destination -L ${DOCS_DIR}/{} {} \;
+	@touch $(BUILDDIR)/.docsrc.sync.ok
+
+.PHONY: copy-src
+copy-src: ${BUILDDIR}/.docsrc.sync.ok
+	@echo "Syncing changed files..."
+	@cd $(DOCS_DIR) && find . -type f -not -path '*/_scripts/*' -newer $(BUILDDIR)/.docsrc.sync.ok -exec cp {} ${BUILDDIR_SRC}/{} \;
+	@cd ${DOCS_DIR} && find . -type l -not -path '*/_scripts/*' -newer $(BUILDDIR)/.docsrc.sync.ok -exec cp --remove-destination -L ${DOCS_DIR}/{} ${BUILDDIR_SRC}/{} \;
+	@cd ${DOCS_DIR} && find -L . -type f -not -path '*/_scripts/*' -newer $(BUILDDIR)/.docsrc.sync.ok -exec cp --remove-destination -L ${DOCS_DIR}/{} ${BUILDDIR_SRC}/{} \;
+	@touch $(BUILDDIR)/.docsrc.sync.ok
 
 .PHONY: generate
-generate: siphon includes-render template-index featurelist
+generate: copy-src clean-siphons $(DOCS_GENERATED_DIR)/.siphon.done $(DOCS_GENERATED_DIR)/.includes-render.done $(DOCS_GENERATED_DIR)/.template-index.done $(DOCS_GENERATED_DIR)/.featurelist.done
 
 .PHONY: clean
 clean:
-	@rm -rf $(BR)/.siphon.dep
+	@rm -rf $(BUILDDIR)
 	@rm -rf $(SCRIPTS_DIR)/siphon/__pycache__
 
diff --git a/docs/_scripts/siphon-generate b/docs/_scripts/siphon-generate
index 9b69c52cf17..f0087f30043 100755
--- a/docs/_scripts/siphon-generate
+++ b/docs/_scripts/siphon-generate
@@ -24,7 +24,7 @@ import os
 import siphon
 
 DEFAULT_LOGFILE = None
-DEFAULT_LOGLEVEL = "info"
+DEFAULT_LOGLEVEL = "warning"
 DEFAULT_OUTPUT = "build-root/docs/siphons"
 DEFAULT_PREFIX = os.getcwd()
 
diff --git a/docs/_scripts/siphon-process b/docs/_scripts/siphon-process
index cbee1e90786..7cc713c515b 100755
--- a/docs/_scripts/siphon-process
+++ b/docs/_scripts/siphon-process
@@ -25,7 +25,7 @@ import sys
 import siphon
 
 DEFAULT_LOGFILE = None
-DEFAULT_LOGLEVEL = "info"
+DEFAULT_LOGLEVEL = "warning"
 DEFAULT_SIPHON = "clicmd"
 DEFAULT_FORMAT = "markdown"
 DEFAULT_OUTPUT = None