doc: clean up PDF generation for ACRN docs

PRs #5945 and #5949 introduced fixes to the doc building process to
support PDF generation of the documentation set.  This PR refines the
doc build process, cleaning up the Makefile, adding display of tool
version information, and updates the doc building documentation to
include additional dependencies needed for building the PDF and
instructions for how to build the PDF.  The latexpdf make target is
provided to just run the latex and PDF producing process that depends on
the HTML artifacts from a make html run.  A new make pdf target is
provided that combines the two steps into one.

A new know-issues pattern file is added that verifies the expected
output from the latexpdf process is returned, as it can't be completely
eliminated without losing potential error messages that need to be
resolved.

Signed-off-by: David B. Kinder <david.b.kinder@intel.com>

doc/.known-issues/doc/pdf.conf

@@ -0,0 +1,23 @@
+# Known PDF generation messages to ignore
+#
+#
+^WARNING: Not copying tabs assets! Not compatible with latex builder
+^Latexmk: Run number 1 of rule 'pdflatex'
+^This is XeTeX, Version .* \(preloaded format=xelatex\)
+^ restricted \\write18 enabled.
+^entering extended mode
+^Latexmk: Summary of warnings:
+^  Latex failed to resolve [0-9]+ reference\(s\)
+^  Latex failed to resolve [0-9]+ citation\(s\)
+#
+^Latexmk: Run number 1 of rule 'makeindex acrn.idx'
+^Latexmk: calling xindy\( -L english -C utf8 -M sphinx.xdy -q -o acrn.ind acrn.idx \)
+#
+^Latexmk: Run number [2-4] of rule 'pdflatex'
+^This is XeTeX, Version .* \(preloaded format=xelatex\)
+^ restricted \\write18 enabled.
+^entering extended mode
+#
+^Latexmk: Run number [2-4] of rule 'makeindex acrn.idx'
+^Latexmk: calling xindy\( -L english -C utf8 -M sphinx.xdy -q -o acrn.ind acrn.idx \)
+#

doc/Makefile

@@ -12,7 +12,8 @@ SPHINXOPTS    ?= -q
 SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = "Project ACRN"
 BUILDDIR      ?= _build
-SOURCEDIR     = $(BUILDDIR)/rst/
+SOURCEDIR     = $(BUILDDIR)/rst
+LATEXMKOPTS   = -silent
 
 # document publication assumes the folder structure is setup
 # with the acrn-hypervisor and projectacrn.github.io repos as
@@ -37,32 +38,45 @@ help:
 
 # Generate the doxygen xml (for Sphinx) and copy the doxygen html to the
 # api folder for publishing along with the Sphinx-generated API docs.
-
-copy-to-sourcedir:
-	$(Q)mkdir -p $(SOURCEDIR)
-	$(Q)rsync -rt --exclude=$(BUILDDIR) . $(SOURCEDIR)
-
-doxy: copy-to-sourcedir
+# This is where we tweak the "pre" API comments into "preconditions"
+# Note that this step starts with an empty doc.log while the others append
+doxy:
 	$(Q)(cat acrn.doxyfile ; echo "OUTPUT_DIRECTORY=$(SOURCEDIR)/doxygen" ) | doxygen - > $(BUILDDIR)/doc.log 2>&1
 	$(Q)find $(SOURCEDIR)/doxygen/xml/* | xargs sed -i 's/simplesect kind="pre"/simplesect kind="preconditions"/'
 
-content: copy-to-sourcedir
+# Copy all the rst content (and images, etc) into the _build/rst folder
+# including rst content and xsd files from the /misc folder that we'll
+# use to generate config option documentation
+
+content:
+	$(Q)mkdir -p $(SOURCEDIR)
+	$(Q)rsync -rt --exclude=$(BUILDDIR) . $(SOURCEDIR)
 	$(Q)scripts/extract_content.py $(SOURCEDIR) misc
 	$(Q)mkdir -p $(SOURCEDIR)/misc/config_tools/schema
 	$(Q)rsync -rt ../misc/config_tools/schema/*.xsd $(SOURCEDIR)/misc/config_tools/schema
 	$(Q)xsltproc -xinclude ./scripts/configdoc.xsl $(SOURCEDIR)/misc/config_tools/schema/config.xsd > $(SOURCEDIR)/reference/configdoc.txt
 
+# Used to pull the acrn kernel source (for API docs)
+
 pullsource:
 	$(Q)scripts/pullsource.sh
 
-html: copy-to-sourcedir doxy content
-	-$(Q)$(SPHINXBUILD) -t $(DOC_TAG) -b html -d $(BUILDDIR)/doctrees $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) $(OPTS) >> $(BUILDDIR)/doc.log 2>&1
+html: content doxy
+	@echo making HTML content
+	$(Q)./scripts/show-versions.py
+	$(Q)$(SPHINXBUILD) -t $(DOC_TAG) -b html -d $(BUILDDIR)/doctrees $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) $(OPTS) >> $(BUILDDIR)/doc.log 2>&1
 	$(Q)./scripts/filter-doc-log.sh $(BUILDDIR)/doc.log
 
-singlehtml: doxy content
-	-$(Q)$(SPHINXBUILD) -t $(DOC_TAG) -b singlehtml -d $(BUILDDIR)/doctrees $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) $(OPTS) >> $(BUILDDIR)/doc.log 2>&1
+singlehtml: content doxy
+	$(Q)$(SPHINXBUILD) -t $(DOC_TAG) -b singlehtml -d $(BUILDDIR)/doctrees $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) $(OPTS) >> $(BUILDDIR)/doc.log 2>&1
 	$(Q)./scripts/filter-doc-log.sh $(BUILDDIR)/doc.log
 
+pdf: html
+	@echo now making $(BUILDDIR)/latex/acrn.pdf
+	$(Q)make -silent latexpdf LATEXMKOPTS=$(LATEXMKOPTS) >> $(BUILDDIR)/doc.log 2>&1
+	$(Q)./scripts/filter-doc-log.sh $(BUILDDIR)/doc.log
+
+
 # Remove generated content (Sphinx and doxygen)
 
 clean:
@@ -72,7 +86,10 @@ clean:
 
 
 # Copy material over to the GitHub pages staging repo
-# along with a README
+# along with a README, index.html redirect to latest/index.html, robots.txt (for
+# search exclusions), and tweak the Sphinx-generated 404.html to work as the
+# site-wide 404 response page.  (We generate the 404.html with Sphinx so it has
+# the current left navigation contents and overall style.)
 
 publish:
 	mkdir -p $(PUBLISHDIR)

doc/conf.py

@@ -293,7 +293,25 @@ latex_elements = {
     #
     'preamble': r'''
 \setcounter{tocdepth}{3}
+\renewcommand\_{\textunderscore\allowbreak}
+\usepackage{listings}
+\usepackage{xcolor}
+\definecolor{IntelMNBlue}{HTML}{003C71}
+\usepackage{titlesec}
+\title{\normalfont\\color{IntelMNBlue}}
+\usepackage{colortbl}
+\protected\def\sphinxstyletheadfamily{\cellcolor[HTML]{DCDCDC}\sffamily\bfseries\color{IntelMNBlue}}
 ''',
+    'sphinxsetup': 'hmargin={0.7in,0.7in}, vmargin={1in,1in},\
+verbatimwithframe=true,\
+verbatimwrapslines=true,\
+TitleColor={HTML}{003C71},\
+HeaderFamily=\\rmfamily\\bfseries, \
+InnerLinkColor={HTML}{003C71},\
+OuterLinkColor={HTML}{003C71},\
+VerbatimColor={HTML}{F0F0F0},\
+VerbatimHighlightColor={HTML}{76CEFF},\
+VerbatimBorderColor={HTML}{00285A}',
 
     # Latex figure (float) alignment
     #
@@ -308,6 +326,7 @@ latex_documents = [
      u'Project ACRN', 'manual'),
 ]
 
+latex_logo = 'images/ACRN_Logo_PrimaryLockup_COLOR-300x300-1.png'
 
 # -- Options for manual page output ---------------------------------------
 

doc/scripts/pullsource.sh

@@ -2,7 +2,8 @@
 # Copyright (C) 2019 Intel Corporation.
 # SPDX-License-Identifier: BSD-3-Clause
 
-q="--quiet"
+#q="--quiet"
+q=""
 
 # get the latest acrn-kernel sources for the kernel-doc API processing
 

doc/scripts/show-versions.py

@@ -24,7 +24,7 @@ class color:
    END = '\033[0m'
 
 # Check all requirements listed in requirements.txt and print out version installed (if any)
-print ("Listing versions of doc build dependencies found on your system...\n")
+print ("doc build tool versions found on your system...\n")
 
 rf = open(os.path.join(sys.path[0], "requirements.txt"),"r")
 
@@ -35,7 +35,7 @@ for reqs in pkg_resources.parse_requirements(rf):
     except:
         print (color.RED + color.BOLD + reqs.project_name + " is missing." + color.END +
                 " (Hint: install all dependencies with " + color.YELLOW +
-                "\"pip3 install --user -r requirements.txt\"" + color.END + ")")
+                "\"pip3 install --user -r scripts/requirements.txt\"" + color.END + ")")
 
 rf.close()
 

doc/tutorials/docbuild.rst

@@ -139,10 +139,10 @@ different (newer) version of doxygen noted above that may also work.
 
 For Ubuntu use:
 
-  .. code-block:: bash
+.. code-block:: bash
 
-     sudo apt install doxygen python3-pip \
-       python3-wheel make graphviz xsltproc
+   sudo apt install doxygen python3-pip \
+     python3-wheel make graphviz xsltproc
 
 Then use ``pip3`` to install the remaining Python-based tools:
 
@@ -174,10 +174,24 @@ And with that you're ready to generate the documentation.
 
       doc/scripts/show-versions.py
 
+
+Extra Tools for Generating PDF
+==============================
+
+While the primary output for ACRN documentation is for an HTML website, you can
+also generate a PDF version of the documentation.  A few extra tools are needed
+only if you want to create PDF output.  For Ubuntu users add these additional
+packages:
+
+.. code-block:: bash
+
+   sudo apt install texlive-latex-recommended texlive-fonts-recommended \
+     texlive-latex-extra latexmk texlive-xetex xindy
+
 Documentation Presentation Theme
 ********************************
 
-Sphinx supports easy customization of the generated documentation
+Sphinx supports easy customization of the generated HTML documentation
 appearance through the use of themes.  Replace the theme files and do
 another ``make html`` and the output layout and style is changed. The
 sphinx build system creates document cache information that attempts to
@@ -196,11 +210,13 @@ Run the Documentation Processors
 
 The ``acrn-hypervisor/doc`` directory has all the ``.rst`` source files, extra
 tools, and ``Makefile`` for generating a local copy of the ACRN technical
-documentation.  For generating all the API documentation, there is a
+documentation. (Some additional ``.rst`` files and other material is extracted
+or generated from the ``/misc`` folder as part of the ``Makefile``.)
+For generating all the API documentation, there is a
 dependency on having the ``acrn-kernel`` repo's contents available too
 (as described previously).  You'll get a sphinx warning if that repo is
 not set up as described, but you can ignore that warning if you're
-not planning to publish.
+not planning to publish or show the API documentation.
 
 .. code-block:: bash
 
@@ -221,6 +237,36 @@ with the command:
 
 and use your web browser to open the URL:  ``http://localhost:8000``.
 
+Generate PDF Output
+===================
+
+After the HTML content is generated, it leaves artifacts behind that you can
+use to generate PDF output using the Sphinx ``latex`` builder.  This
+builder initially creates LaTeX output in the ``_build/latex`` folder and then
+uses the ``latexmk`` tool to create the final ``acrn.pdf`` file in the same
+folder.  This process automatically makes a few passes over the content to create the index
+and resolve intra-document hyperlinks, and produces plenty of progress messages along the
+way.  The Sphinx-generated output for tables and code blocks also yields many "underfill"
+and "overfill" messages from the ``latex`` builder that can't be easily
+suppressed or fixed.  Because of that, we recommend running this PDF generation
+with the ``-silent`` options, like this (after you've run ``make html``:
+
+.. code-block:: bash
+
+   make latexpdf LATEXMKOPTS="-silent"
+
+For convenience, we've also created a make target called ``pdf`` that will first
+generate the HTML content and then make the PDF file in one step:
+
+.. code-block:: bash
+
+   make pdf
+
+This make target runs quietly, and then verifies that no unexpected message from
+the build process are produced (using the :ref:`message filtering process
+<filter_expected>` explained below.  Either way, when the build completes, the
+generated PDF file is in ``_build/latex/acrn.pdf``.
+
 Publish Content
 ***************
 
@@ -299,6 +345,8 @@ of the repo, and add some extra flags to the ``make`` commands:
    make DOC_TAG=release RELEASE=2.3 html
    make DOC_TAG=release RELEASE=2.3 publish
 
+.. _filter_expected:
+
 Filter Expected Warnings
 ************************