gh-137574: Add Support For Special Names in help (GH-137966)

Author: adqm

Doc/library/__main__.rst

@@ -1,3 +1,5 @@
+.. _`__main__`:
+
 :mod:`!__main__` --- Top-level code environment
 ===============================================
 

Doc/reference/datamodel.rst

@@ -2992,6 +2992,8 @@ automatic property creation, proxies, frameworks, and automatic resource
 locking/synchronization.
 
 
+.. _customize-instance-subclass-checks:
+
 Customizing instance and subclass checks
 ----------------------------------------
 

Doc/tools/extensions/pydoc_topics.py

@@ -41,12 +41,14 @@
     "callable-types",
     "calls",
     "class",
+    "class-customization",
     "comparisons",
     "compound",
     "context-managers",
     "continue",
     "conversions",
     "customization",
+    "customize-instance-subclass-checks",
     "debugger",
     "del",
     "dict",
@@ -70,6 +72,7 @@
     "lambda",
     "lazy",
     "lists",
+    "name_equals_main",
     "match",
     "naming",
     "nonlocal",
@@ -102,6 +105,7 @@
     "while",
     "with",
     "yield",
+    "__main__",
 })
 
 

Doc/whatsnew/3.16.rst

@@ -614,6 +614,9 @@ pydoc
   inherited from other classes are collapsed by default.
   (Contributed by Serhiy Storchaka in :gh:`153906`.)
 
+* Add help for more special names to the built-in :func:`help` function.
+  (Contributed by Adam Hartz in :gh:`137966`.)
+
 
 pickle
 ------

Lib/pydoc.py

@@ -68,6 +68,7 @@ class or function within a module or module in a package.  If the
 import textwrap
 import time
 import tokenize
+import urllib.parse
 import warnings
 from annotationlib import Format
 from collections import deque
@@ -1709,6 +1710,14 @@ def locate(path, forceload=0):
             object = getattr(object, part)
         except AttributeError:
             return None
+    if (
+        _is_dunder_name(path)
+        and not isinstance(object, (type, type(__import__), type(__future__)))
+    ):
+        # if we're looking up a special variable and we don't find a class,
+        # function, or module, it's probably not what the user wanted (if it
+        # is, they can look up builtins.whatever)
+        return None
     return object
 
 # --------------------------------------- interactive interpreter interface
@@ -1722,10 +1731,15 @@ def resolve(thing, forceload=0):
     if isinstance(thing, str):
         object = locate(thing, forceload)
         if object is None:
+            if _is_dunder_name(thing):
+                special = "Use help('specialnames') for a list of special names for which help is available.\n"
+            else:
+                special = ""
             raise ImportError('''\
-No Python documentation found for %r.
-Use help() to get the interactive help utility.
-Use help(str) for help on the str class.''' % thing)
+No help entry found for %r.
+%sUse help() to get the interactive help utility.
+Use help(str) for help on the str class.
+Additional documentation is available online at https://docs.python.org/%s.%s/''' % (thing, special, *sys.version_info[:2]))
         return object, thing
     else:
         name = getattr(thing, '__name__', None)
@@ -1816,10 +1830,10 @@ def _introdoc():
         Python, you should definitely check out the tutorial at
         https://docs.python.org/{ver}/tutorial/.
 
-        Enter the name of any module, keyword, or topic to get help on writing
-        Python programs and using Python modules.  To get a list of available
-        modules, keywords, symbols, or topics, enter "modules", "keywords",
-        "symbols", or "topics".
+        Enter the name of any module, keyword, symbol, or topic to get help on
+        writing Python programs and using Python modules.  To get a list of
+        available modules, keywords, symbols, special names, or topics, enter
+        "modules", "keywords", "symbols", "specialnames", or "topics".
         {pyrepl_keys}
         Each module also comes with a one-line summary of what it does; to list
         the modules whose name or summary contain a given string such as "spam",
@@ -1829,6 +1843,96 @@ def _introdoc():
         enter "q", "quit" or "exit".
     ''')
 
+def _is_dunder_name(x):
+    return isinstance(x, str) and x.isidentifier() and len(x) > 4 and x[:2] == x[-2:] == '__'
+
+def collect_dunders(symbols):
+    dunders = {
+        '__name__': ('name_equals_main', ''),
+        '__main__': ('__main__', ''),
+        '__call__': ('callable-types', 'SPECIALMETHODS'),
+    }
+
+    basic_dunders = [
+        '__new__', '__init__', '__del__', '__repr__', '__str__', '__bytes__',
+        '__format__', '__hash__', '__bool__',
+    ]
+    for bd in basic_dunders:
+        dunders[bd] = ('customization', 'SPECIALMETHODS')
+
+    attribute_dunders = [
+        '__getattr__', '__getattribute__', '__setattr__', '__delattr__',
+        '__dir__', '__get__', '__set__', '__delete__', '__objclass__',
+    ]
+    for ad in attribute_dunders:
+        dunders[ad] = ('attribute-access', 'SPECIALMETHODS')
+
+    class_dunders = [
+        '__init_subclass__', '__set_name__', '__mro_entries__',
+    ]
+    for cd in class_dunders:
+        dunders[cd] = ('class-customization', 'SPECIALMETHODS')
+
+    instance_dunders = [
+        '__instancecheck__', '__subclasscheck__'
+    ]
+    for d in instance_dunders:
+        dunders[d] = ('customize-instance-subclass-checks', 'SPECIALMETHODS')
+
+    sequence_dunders = [
+       '__len__', '__length_hint__', '__getitem__', '__setitem__',
+       '__delitem__', '__missing__', '__iter__', '__reversed__',
+       '__contains__',
+    ]
+    for sd in sequence_dunders:
+        dunders[sd] = ('sequence-types', 'SPECIALMETHODS')
+
+    comparison_dunders = {
+        '__lt__': '<',
+        '__le__': '<=',
+        '__eq__': '==',
+        '__ne__': '!=',
+        '__gt__': '>',
+        '__ge__': '>=',
+    }
+    for dunder, symbol in comparison_dunders.items():
+        dunders[dunder] = ('customization', f'{symbol} SPECIALMETHODS')
+        if symbol in symbols:
+            symbols[symbol] += f' {dunder}'
+
+    arithmetic_dunders = {
+        '__add__': '+',
+        '__sub__': '-',
+        '__mul__': '*',
+        '__truediv__': '/',
+        '__floordiv__': '//',
+        '__mod__': '%',
+        '__pow__': '**',
+        '__lshift__': '<<',
+        '__rshift__': '>>',
+        '__and__': '&',
+        '__or__': '|',
+        '__xor__': '^',
+    }
+    for dunder, symbol in arithmetic_dunders.items():
+        rname = "__r" + dunder[2:]
+        iname = "__i" + dunder[2:]
+        dunders[dunder] = ('numeric-types', f'{symbol} {rname} {iname} SPECIALMETHODS')
+        dunders[rname] = ('numeric-types', f'{symbol} {dunder} SPECIALMETHODS')
+        dunders[iname] = ('numeric-types', f'{symbol} {dunder} SPECIALMETHODS')
+        if symbol in symbols:
+            symbols[symbol] += f' {dunder}'
+
+    # __matmul__ isn't included above because help('@') doesn't talk about
+    # matrix multiplication, so we shouldn't list it here as a related topic.
+    dunders['__matmul__'] = ('numeric-types', f'__rmatmul__ __imatmul__ SPECIALMETHODS')
+    dunders['__rmatmul__'] = ('numeric-types', f'__matmul__ SPECIALMETHODS')
+    dunders['__imatmul__'] = ('numeric-types', f'__matmul__ SPECIALMETHODS')
+
+    dunders['__divmod__'] = ('numeric-types', 'divmod')
+
+    return dunders
+
 class Helper:
 
     # These dictionaries map a topic name to either an alias, or a tuple
@@ -1912,7 +2016,8 @@ class Helper:
         '(': 'TUPLES FUNCTIONS CALLS',
         ')': 'TUPLES FUNCTIONS CALLS',
         '[': 'LISTS SUBSCRIPTS SLICINGS',
-        ']': 'LISTS SUBSCRIPTS SLICINGS'
+        ']': 'LISTS SUBSCRIPTS SLICINGS',
+
     }
     for topic, symbols_ in _symbols_inverse.items():
         for symbol in symbols_:
@@ -1962,8 +2067,7 @@ class Helper:
         'BASICMETHODS': ('customization', 'hash repr str SPECIALMETHODS'),
         'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'),
         'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'),
-        'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS '
-                             'SPECIALMETHODS'),
+        'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SPECIALMETHODS'),
         'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'),
         'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT '
                           'SPECIALMETHODS'),
@@ -2008,8 +2112,15 @@ class Helper:
         'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'),
         'DEBUGGING': ('debugger', 'pdb'),
         'CONTEXTMANAGERS': ('context-managers', 'with'),
+        'DUNDERMETHODS': 'SPECIALMETHODS',
+        'MAINMODULE': '__main__',
     }
 
+    # add dunder methods
+    dunders = collect_dunders(symbols)
+    topics |= dunders
+
+
     def __init__(self, input=None, output=None):
         self._input = input
         self._output = output
@@ -2083,6 +2194,8 @@ def help(self, request, is_cli=False):
             if request == 'keywords': self.listkeywords()
             elif request == 'symbols': self.listsymbols()
             elif request == 'topics': self.listtopics()
+            elif request == 'specialnames':
+                self.listdunders()
             elif request == 'modules': self.listmodules()
             elif request[:8] == 'modules ':
                 self.listmodules(request.split()[1])
@@ -2134,7 +2247,14 @@ def listtopics(self):
 Here is a list of available topics.  Enter any topic name to get more help.
 
 ''')
-        self.list(self.topics.keys(), columns=3)
+        self.list([k for k in self.topics if k not in self.dunders], columns=3)
+
+    def listdunders(self):
+        self.output.write('''
+Here is a list of special names for which help is available.  Enter any one to get more help.
+
+''')
+        self.list(self.dunders.keys(), columns=3)
 
     def showtopic(self, topic, more_xrefs=''):
         try:
@@ -2187,6 +2307,9 @@ def _gettopic(self, topic, more_xrefs=''):
 Sorry, topic and keyword documentation is not available because the
 module "pydoc_data.topics" could not be found.
 ''' , '')
+        if topic in self.symbols:
+            topic, _, xrefs = self.symbols[topic].partition(' ')
+            more_xrefs = ' '.join([xrefs, more_xrefs]).strip()
         target = self.topics.get(topic, self.keywords.get(topic))
         if not target:
             raise ValueError('could not find topic')
@@ -2559,14 +2682,14 @@ def onerror(modname):
 
         # format page
         def bltinlink(name):
-            return '<a href="%s.html">%s</a>' % (name, name)
+            return '<a href="%s.html">%s</a>' % (urllib.parse.quote(name), html.escape(name))
 
         results = []
         heading = html.heading('Search Results')
         for name, desc in search_result:
             results.append('<li>%s%s</li>' % (bltinlink(name), desc))
         contents = heading + html.bigsection(
-            'key = %s' % key, 'index', '<ul>\n%s\n</ul>' % '\n'.join(results))
+            'key = %s' % html.escape(key), 'index', '<ul>\n%s\n</ul>' % '\n'.join(results))
         return 'Search Results', contents
 
     def html_topics():
@@ -2592,13 +2715,27 @@ def bltinlink(name):
         contents = heading + html.multicolumn(names, bltinlink)
         return 'Keywords', contents
 
+    def html_symbols():
+        """Index of symbols available."""
+
+        def bltinlink(name):
+            return '<a href="topic?key=%s">%s</a>' % (urllib.parse.quote(name), html.escape(name))
+
+        heading = html.heading('Symbols')
+        names = sorted(Helper.symbols.keys())
+
+        contents = heading + html.multicolumn(names, bltinlink)
+        return 'Symbols', contents
+
     def html_topicpage(topic):
         """Topic or keyword help page."""
         buf = io.StringIO()
         htmlhelp = Helper(buf, buf)
         contents, xrefs = htmlhelp._gettopic(topic)
         if topic in htmlhelp.keywords:
             title = 'KEYWORD'
+        elif topic in htmlhelp.symbols:
+            title = 'SYMBOL'
         else:
             title = 'TOPIC'
         heading = html.heading('%s %s' % (title.capitalize(), topic))
@@ -2627,7 +2764,7 @@ def html_error(url, exc):
         heading = html.heading('Error')
         contents = '<br>'.join(html.escape(line) for line in
                                format_exception_only(type(exc), exc))
-        contents = heading + html.bigsection(url, 'error', contents)
+        contents = heading + html.bigsection(html.escape(urllib.parse.unquote(url)), 'error', contents)
         return "Error - %s" % url, contents
 
     def get_html_page(url):
@@ -2642,8 +2779,11 @@ def get_html_page(url):
                 title, content = html_topics()
             elif url == "keywords":
                 title, content = html_keywords()
+            elif url == "symbols":
+                title, content = html_symbols()
             elif '=' in url:
                 op, _, url = url.partition('=')
+                url = urllib.parse.unquote(url).strip()
                 if op == "search?key":
                     title, content = html_search(url)
                 elif op == "topic?key":
@@ -2824,7 +2964,8 @@ class BadUsage(Exception): pass
     reference to a class or function within a module or module in a
     package.  If <name> contains a '{sep}', it is used as the path to a
     Python source file to document. If name is 'keywords', 'topics',
-    or 'modules', a listing of these things is displayed.
+    'symbols', 'specialnames', or 'modules', a listing of these things is
+    displayed.
 
 {cmd} -k <keyword>
     Search for a keyword in the synopsis lines of all available modules.

Lib/pydoc_data/_pydoc.css

@@ -127,7 +127,7 @@ section section {
     margin: 0.3em 0;
 }
 
-pre.topic, pre.keyword {
+pre.topic, pre.keyword, pre.symbol {
     background-color: var(--block-background-color);
     border: 1px solid var(--block-border-color);
     border-radius: 3px;