doc: update genrest script for latest kconfiglib

The genrest.py script (used to generate config documentation from the
Kconfig files) broke when upgrading to the latest kconfiglib (10.9.1).
Copied the latest genrest.py script from the Zephyr project (where this
script and processing was developed) and things work again.

Also update requirements.txt to match the kconfiglib version verified to
work.

tracked-on: #1387

Signed-off-by: David B. Kinder <david.b.kinder@intel.com>

doc/scripts/genrest.py

@@ -9,38 +9,41 @@ import textwrap
 
 import kconfiglib
 
-# "Extend" the standard kconfiglib.expr_str() to turn references to defined
-# Kconfig symbols into RST links. Symbol.__str__() will then use the extended
-# version.
-#
-# This is a bit hacky, but better than reimplementing Symbol.__str__() and/or
-# kconfiglib.expr_str().
 
-def expr_str_rst(expr):
+def rst_link(sc):
-    # Skip constant and undefined symbols by checking if expr.nodes is empty
+    # Returns an RST link (string) for the symbol/choice 'sc', or the normal
-    if isinstance(expr, kconfiglib.Symbol) and expr.nodes:
+    # Kconfig expression format (e.g. just the name) for 'sc' if it can't be
-        # The "\ " avoids RST issues for !CONFIG_FOO -- see
+    # turned into a link.
-        # http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#character-level-inline-markup
-        return r"\ :option:`{0} <CONFIG_{0}>`".format(expr.name)
 
-    # Choices appear as dependencies of choice symbols.
+    if isinstance(sc, kconfiglib.Symbol):
-    #
+        # Skip constant and undefined symbols by checking if expr.nodes is
-    # Use a :ref: instead of an :option:. With an :option:, we'd have to have
+        # empty
-    # an '.. option::' in the choice reference page as well. That would make
+        if sc.nodes:
-    # the internal choice ID show up in the documentation.
+            # The "\ " avoids RST issues for !CONFIG_FOO -- see
-    #
+            # http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#character-level-inline-markup
-    # Note that the first pair of <...> is non-syntactic here. We just display
+            return r"\ :option:`{0} <CONFIG_{0}>`".format(sc.name)
-    # choices links within <> in the documentation.
+
-    if isinstance(expr, kconfiglib.Choice):
+    elif isinstance(sc, kconfiglib.Choice):
+        # Choices appear as dependencies of choice symbols.
+        #
+        # Use a :ref: instead of an :option:. With an :option:, we'd have to have
+        # an '.. option::' in the choice reference page as well. That would make
+        # the internal choice ID show up in the documentation.
+        #
+        # Note that the first pair of <...> is non-syntactic here. We just display
+        # choices links within <> in the documentation.
         return r"\ :ref:`<{}> <{}>`" \
-               .format(choice_desc(expr), choice_id(expr))
+               .format(choice_desc(sc), choice_id(sc))
 
-    # We'll end up back in expr_str_rst() when expr_str_orig() does recursive
+    # Can't turn 'sc' into a link. Use the standard Kconfig format.
-    # calls for subexpressions
+    return kconfiglib.standard_sc_expr_str(sc)
-    return expr_str_orig(expr)
 
-expr_str_orig = kconfiglib.expr_str
+
-kconfiglib.expr_str = expr_str_rst
+def expr_str(expr):
+    # Returns the Kconfig representation of 'expr', with symbols/choices turned
+    # into RST links
+
+    return kconfiglib.expr_str(expr, rst_link)
 
 
 INDEX_RST_HEADER = """.. _configuration:
@@ -87,12 +90,8 @@ def write_kconfig_rst():
     # String with the RST for the index page
     index_rst = INDEX_RST_HEADER
 
-    # - Sort the symbols by name so that they end up in sorted order in
+    # Sort the symbols by name so that they end up in sorted order in index.rst
-    #   index.rst
+    for sym in sorted(kconf.unique_defined_syms, key=lambda sym: sym.name):
-    #
-    # - Use set() to get rid of duplicates for symbols defined in multiple
-    #   locations.
-    for sym in sorted(set(kconf.defined_syms), key=lambda sym: sym.name):
         # Write an RST file for the symbol
         write_sym_rst(sym, out_dir)
 
@@ -104,7 +103,7 @@ def write_kconfig_rst():
             " / ".join(node.prompt[0]
                        for node in sym.nodes if node.prompt))
 
-    for choice in kconf.choices:
+    for choice in kconf.unique_choices:
         # Write an RST file for the choice
         write_choice_rst(choice, out_dir)
 
@@ -120,6 +119,7 @@ def write_sym_rst(sym, out_dir):
                      direct_deps_rst(sym) +
                      defaults_rst(sym) +
                      select_imply_rst(sym) +
+                     selecting_implying_rst(sym) +
                      kconfig_definition_rst(sym))
 
 
@@ -202,24 +202,39 @@ def direct_deps_rst(sc):
            "===================\n\n" \
            "{}\n\n" \
            "*(Includes any dependencies from if's and menus.)*\n\n" \
-           .format(kconfiglib.expr_str(sc.direct_dep))
+           .format(expr_str(sc.direct_dep))
 
 
 def defaults_rst(sc):
     # Returns RST that lists the 'default' properties of 'sc' (symbol or
     # choice)
 
-    if not sc.defaults:
+    if isinstance(sc, kconfiglib.Symbol) and sc.choice:
+        # 'default's on choice symbols have no effect (and generate a warning).
+        # The implicit value hint below would be misleading as well.
         return ""
 
     rst = "Defaults\n" \
           "========\n\n"
 
-    for value, cond in sc.defaults:
+    if sc.defaults:
-        default_str = kconfiglib.expr_str(value)
+        for value, cond in sc.defaults:
-        if cond is not sc.kconfig.y:
+            rst += "- " + expr_str(value)
-            default_str += " if " + kconfiglib.expr_str(cond)
+            if cond is not sc.kconfig.y:
-        rst += "- {}\n".format(default_str)
+                rst += " if " + expr_str(cond)
+            rst += "\n"
+
+    else:
+        rst += "No defaults. Implicitly defaults to "
+
+        if isinstance(sc, kconfiglib.Choice):
+            rst += "the first (visible) choice option.\n"
+        elif sc.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE):
+            rst += "``n``.\n"
+        else:
+            # This is accurate even for int/hex symbols, though an active
+            # 'range' might clamp the value (which is then treated as zero)
+            rst += "the empty string.\n"
 
     return rst + "\n"
 
@@ -235,53 +250,95 @@ def choice_syms_rst(choice):
 
     for sym in choice.syms:
         # Generates a link
-        rst += "- {}\n".format(kconfiglib.expr_str(sym))
+        rst += "- {}\n".format(expr_str(sym))
 
     return rst + "\n"
 
 
 def select_imply_rst(sym):
+    # Returns RST that lists the symbols 'select'ed or 'imply'd by the symbol
+
+    rst = ""
+
+    def add_select_imply_rst(type_str, lst):
+        # Adds RST that lists the selects/implies from 'lst', which holds
+        # (<symbol>, <condition>) tuples, if any. Also adds a heading derived
+        # from 'type_str' if there any selects/implies.
+
+        nonlocal rst
+
+        if lst:
+            heading = "Symbols {} by this symbol".format(type_str)
+            rst += "{}\n{}\n\n".format(heading, len(heading)*"=")
+
+            for select, cond in lst:
+                rst += "- " + rst_link(select)
+                if cond is not sym.kconfig.y:
+                    rst += " if " + expr_str(cond)
+                rst += "\n"
+
+            rst += "\n"
+
+    add_select_imply_rst("selected", sym.selects)
+    add_select_imply_rst("implied", sym.implies)
+
+    return rst
+
+
+def selecting_implying_rst(sym):
     # Returns RST that lists the symbols that are 'select'ing or 'imply'ing the
     # symbol
 
     rst = ""
 
-    def add_select_imply_rst(type_str, expr):
+    def add_selecting_implying_rst(type_str, expr):
-        # Writes a link for each selecting symbol (if 'expr' is sym.rev_dep) or
+        # Writes a link for each symbol that selects the symbol (if 'expr' is
-        # each implying symbol (if 'expr' is sym.weak_rev_dep). Also adds a
+        # sym.rev_dep) or each symbol that imply's the symbol (if 'expr' is
-        # heading at the top, derived from type_str ("select"/"imply").
+        # sym.weak_rev_dep). Also adds a heading at the top derived from
+        # type_str ("select"/"imply"), if there are any selecting/implying
+        # symbols.
 
         nonlocal rst
 
-        heading = "Symbols that ``{}`` this symbol".format(type_str)
+        if expr is not sym.kconfig.n:
-        rst += "{}\n{}\n\n".format(heading, len(heading)*"=")
+            heading = "Symbols that {} this symbol".format(type_str)
+            rst += "{}\n{}\n\n".format(heading, len(heading)*"=")
 
-        # The reverse dependencies from each select/imply are ORed together
+            # The reverse dependencies from each select/imply are ORed together
-        for select in kconfiglib.split_expr(expr, kconfiglib.OR):
+            for select in kconfiglib.split_expr(expr, kconfiglib.OR):
-            # - 'select/imply A if B' turns into A && B
+                # - 'select/imply A if B' turns into A && B
-            # - 'select/imply A' just turns into A
+                # - 'select/imply A' just turns into A
-            #
+                #
-            # In both cases, we can split on AND and pick the first
+                # In both cases, we can split on AND and pick the first
-            # operand.
+                # operand.
 
-            # kconfiglib.expr_str() generates a link
+                rst += "- {}\n".format(rst_link(
-            rst += "- {}\n".format(kconfiglib.expr_str(
+                    kconfiglib.split_expr(select, kconfiglib.AND)[0]))
-                kconfiglib.split_expr(select, kconfiglib.AND)[0]))
 
-        rst += "\n"
+            rst += "\n"
 
-    if sym.rev_dep is not sym.kconfig.n:
+    add_selecting_implying_rst("select", sym.rev_dep)
-        add_select_imply_rst("select", sym.rev_dep)
+    add_selecting_implying_rst("imply", sym.weak_rev_dep)
-
-    if sym.weak_rev_dep is not sym.kconfig.n:
-        add_select_imply_rst("imply", sym.weak_rev_dep)
 
     return rst
 
 
 def kconfig_definition_rst(sc):
-    # Returns RST that lists the Kconfig definition(s) of 'sc' (symbol or
+    # Returns RST that lists the Kconfig definition location, include path,
-    # choice)
+    # menu path, and Kconfig definition for each node (definition location) of
+    # 'sc' (symbol or choice)
+
+    # Fancy Unicode arrow. Added in '93, so ought to be pretty safe.
+    arrow = " \N{RIGHTWARDS ARROW} "
+
+    def include_path(node):
+        if not node.include_path:
+            # In the top-level Kconfig file
+            return ""
+
+        return "Included via {}\n\n".format(
+            arrow.join("``{}:{}``".format(filename, linenr)
+                       for filename, linenr in node.include_path))
 
     def menu_path(node):
         path = ""
@@ -299,8 +356,7 @@ def kconfig_definition_rst(sc):
             if node is node.kconfig.top_node:
                 break
 
-            # Fancy Unicode arrow. Added in '93, so ought to be pretty safe.
+            path = arrow + node.prompt[0] + path
-            path = " → " + node.prompt[0] + path
 
         return "(top menu)" + path
 
@@ -308,14 +364,22 @@ def kconfig_definition_rst(sc):
     if len(sc.nodes) > 1: heading += "s"
     rst = "{}\n{}\n\n".format(heading, len(heading)*"=")
 
-    rst += ".. highlight:: kconfig\n\n"
+    rst += ".. highlight:: kconfig"
 
-    rst += "\n\n".join(
+    for node in sc.nodes:
-        "At ``{}:{}``, in menu ``{}``:\n\n"
+        rst += "\n\n" \
-        ".. parsed-literal::\n\n"
+               "At ``{}:{}``\n\n" \
-        "{}".format(node.filename, node.linenr, menu_path(node),
+               "{}" \
-                    textwrap.indent(str(node), " "*4))
+               "Menu path: {}\n\n" \
-        for node in sc.nodes)
+               ".. parsed-literal::\n\n{}" \
+               .format(node.filename, node.linenr,
+                       include_path(node), menu_path(node),
+                       textwrap.indent(node.custom_str(rst_link), 4*" "))
+
+        # Not the last node?
+        if node is not sc.nodes[-1]:
+            # Add a horizontal line between multiple definitions
+            rst += "\n\n----"
 
     rst += "\n\n*(Definitions include propagated dependencies, " \
            "including from if's and menus.)*"

doc/scripts/requirements.txt

@@ -2,4 +2,4 @@ breathe==4.9.1
 sphinx==1.7.7
 docutils==0.14
 sphinx_rtd_theme==0.4.0
-kconfiglib
+kconfiglib==10.9.1