[PATCH 1/3 v2] docs: Add grub command documentation checks

[Hans Ulrich Niedermann]

This adds the docs/check-commands.py script which checks whether
grub commands implemented in the git source tree are documented
in docs/grub.texi and vice versa.

During a standard "make" command, BUILT_SOURCES makes sure that
docs/undocumented-commands.texi will be created (if it does not
exist yet) or updated (if it differs from what the script would
generate) or be left alone otherwise.

If you run "make grub.info" in the docs/ subdirectory, the
BUILT_SOURCES trick to create or update undocumented-commands.texi
will not work (this is an Automake limitation). So if you want
to avoid the "all" make target, you need to explictly run
"make update-undoc" first.

The build time requirements are already required by other parts
of the grub buildsystem:

  * $(CMP)
    compare two files by content

  * $(PYTHON)
    check-commands.py has been written to run on Python 2 and 3
    (tested with Python 2.7 and Python 3.7)

The time docs/check-commands.py needs to read and basically grep
through docs/grub.texi and all *.c source files is quick enough
to not significantly slow down a "make" on the grub source tree.

Signed-off-by: Hans Ulrich Niedermann <address@hidden>
---
 .gitignore             |   1 +
 docs/Makefile.am       |  16 +++-
 docs/check-commands.py | 173 +++++++++++++++++++++++++++++++++++++++++
 docs/grub.texi         |   4 +
 4 files changed, 191 insertions(+), 3 deletions(-)
 create mode 100644 docs/check-commands.py

diff --git a/.gitignore b/.gitignore
index 149b3713a..bdedd4e89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ DISTLIST
 docs/*.info
 /docs/*.info-*
 docs/stamp-vti
+/docs/undocumented-commands.texi
 docs/version.texi
 ehci_test
 example_grub_script_test
diff --git a/docs/Makefile.am b/docs/Makefile.am
index 93eb39627..8e8500226 100644
--- a/docs/Makefile.am
+++ b/docs/Makefile.am
@@ -2,8 +2,18 @@ AUTOMAKE_OPTIONS = subdir-objects
 
 # AM_MAKEINFOFLAGS = --no-split --no-validate
 info_TEXINFOS = grub.texi grub-dev.texi
-grub_TEXINFOS = fdl.texi
-
-EXTRA_DIST = font_char_metrics.png font_char_metrics.txt
+grub_TEXINFOS = fdl.texi undocumented-commands.texi
 
+EXTRA_DIST = check-commands.py font_char_metrics.png font_char_metrics.txt
 
+CLEANFILES = undocumented-commands.texi
+BUILT_SOURCES = update-undoc
+update-undoc:
+       $(PYTHON) check-commands.py > undocumented-commands.texi.tmp
+       @if test -f $(srcdir)/undocumented-commands.texi && $(CMP) 
undocumented-commands.texi.tmp $(srcdir)/undocumented-commands.texi; then \
+               echo "Not updating undocumented-commands.texi: is up to date"; \
+               rm -f undocumented-commands.texi.tmp; \
+       else \
+               echo "Updating undocumented-commands.texi"; \
+               mv -f undocumented-commands.texi.tmp 
$(srcdir)/undocumented-commands.texi; \
+       fi
diff --git a/docs/check-commands.py b/docs/check-commands.py
new file mode 100644
index 000000000..3c99bfbb2
--- /dev/null
+++ b/docs/check-commands.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import re
+
+import sys
+
+
+class CommandChecker:
+
+    def __init__(self):
+        srcdir, self.prog = os.path.split(__file__)
+        self.top_srcdir = os.path.dirname(os.path.abspath(srcdir))
+
+    def read_texi_text_commands(self, texi_filename):
+        texi_pathname = os.path.join(self.top_srcdir, 'docs', texi_filename)
+        r = re.compile(r'@command\{([a-zA-Z0-9_-]+)\}')
+        commands = set()
+        with open(texi_pathname) as texi:
+            for line in texi.readlines():
+                for m in r.finditer(line):
+                    commands.add(m[1])
+        return commands
+
+    def read_texi_command_menu(self, texi_filename):
+        texi_pathname = os.path.join(self.top_srcdir, 'docs', texi_filename)
+        r = re.compile(r'\* ([^:]+)::')
+        n = re.compile(r'^@node .+ commands$')
+        commands = set()
+        waiting_for_node = True
+        waiting_for_menu = False
+        collecting_commands = False
+        with open(texi_pathname) as texi:
+            for line in texi.readlines():
+                line = line.strip()
+                if line.startswith('@comment'):
+                    continue
+                if waiting_for_node:
+                    if n.match(line):
+                        # print("@comment", line)
+                        waiting_for_node = False
+                        waiting_for_menu = True
+                        continue
+                if waiting_for_menu:
+                    if line == '@menu':
+                        waiting_for_menu = False
+                        collecting_commands = True
+                        continue
+                if collecting_commands:
+                    if line == '@end menu':
+                        collecting_commands = False
+                        waiting_for_node = True
+                        continue
+                    m = r.match(line)
+                    # print("@comment  ", m[1])
+                    commands.add(m.group(1))
+        return commands
+
+    def read_src_commands(self):
+        top = os.path.join(self.top_srcdir, 'grub-core')
+        r = 
re.compile(r'grub_register_(command|command_p1|extcmd)\s*\("([a-z0-9A-Z_\[]+)",')
+        commands = set()
+        for dirpath, dirnames, filenames in os.walk(top):
+            for fn in filenames:
+                fp = os.path.join(dirpath, fn)
+                fpe = os.path.splitext(fp)[1]
+                if fpe == '.c':
+                    with open(fp) as cfile:
+                        for line in cfile.readlines():
+                            for m in r.finditer(line):
+                                commands.add(m.group(2))
+        return commands
+
+
+def write_undocumented_commands(commands):
+    print("""\
+@node Undocumented commands
+@section The list of undocumented commands
+""")
+
+    if not commands:
+        print("""
+There appear to be no undocumented commands at this time.
+""")
+        return
+
+    print("""\
+These commands are implemented in the grub software but still need to be
+documented and sorted into categories.
+""")
+
+    maxlen_str = sorted(list(commands), key=lambda cmd: len(cmd),
+                        reverse=True)[0]
+    fmt = '* %%-%ds %%s' % (2+len(maxlen_str))
+
+    print("@menu")
+    for cmd in sorted(list(commands)):
+        print(fmt % ("%s::" % cmd, "Undocumented command"))
+    print("@end menu")
+    print()
+
+    for cmd in sorted(list(commands)):
+        print("@node %s" % cmd)
+        print("@subsection %s" % cmd)
+        print()
+        print("The grub command @command{%s} has not been documented properly 
yet." % cmd)
+        print()
+
+
+def print_set(st):
+    for i, item in enumerate(sorted(list(st)), start=1):
+        print("@comment", "  %d." % i, item)
+
+
+def main():
+    cc = CommandChecker()
+    print("""\
+@comment Automatically generated by %(prog)s
+@comment Do not modify this generated file.
+@comment
+@comment If you want to document some of the commands listed below, feel
+@comment free to copy over some of the @nodes and @subsections below
+@comment into the grub.texi file proper and re-generate this file.
+@comment""" % {'prog': cc.prog})
+
+    # texi_text_commands = cc.read_texi_text_commands('grub.texi')
+    # print("@comment", "Commands in grub.texi text:")
+    # print_set(texi_text_commands)
+
+    texi_menu_commands = cc.read_texi_command_menu('grub.texi')
+    # print("@comment", "Commands in grub.texi menu:")
+    # print_set(texi_menu_commands)
+
+    src_commands = cc.read_src_commands()
+    # print("@comment", "Commands in grub source:")
+    # print_set(src_commands)
+
+    doc_without_src = texi_menu_commands - src_commands
+    if doc_without_src:
+        print("@comment",
+              "Commands documented in a grub.texi menu but not registered in 
grub source:")
+        print_set(doc_without_src)
+    else:
+        print("@comment",
+              "There appear to be no commands documented in a grub.texi menu 
but not")
+        print("@comment",
+              "registered in grub source. Congratulations!")
+
+    print("@comment")
+
+    src_without_doc = src_commands - texi_menu_commands
+    if src_without_doc:
+        print("@comment",
+              "Commands registered in grub source but not documented in a 
grub.texi node:")
+        print_set(src_without_doc)
+
+    print()
+
+    write_undocumented_commands(src_without_doc)
+
+    # Once grub.texi actually documents all commands, we can uncomment
+    # this and actually fail if the set of implemented commands and
+    # the set of documented commands differ in any way.
+    # if ((len(doc_without_src) > 0) or (len(src_without_doc) > 0)):
+    #     sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docs/grub.texi b/docs/grub.texi
index d6408d242..0865ffa17 100644
--- a/docs/grub.texi
+++ b/docs/grub.texi
@@ -3764,6 +3764,7 @@ shell}.
 * General commands::
 * Command-line and menu entry commands::
 * Networking commands::
+* Undocumented commands::
 @end menu
 
 
@@ -5636,6 +5637,9 @@ is given, use default list of servers.
 @end deffn
 
 
+@include undocumented-commands.texi
+
+
 @node Internationalisation
 @chapter Internationalisation
 
-- 
2.25.3