Migrate to MarkupContent and convert docstrings to Markdown (#80)

Author: krassowski

pylsp/_utils.py

@@ -8,7 +8,9 @@
 import pathlib
 import re
 import threading
+from typing import List, Optional
 
+import docstring_to_markdown
 import jedi
 
 JEDI_VERSION = jedi.__version__
@@ -144,17 +146,84 @@ def _merge_dicts_(a, b):
     return dict(_merge_dicts_(dict_a, dict_b))
 
 
-def format_docstring(contents):
-    """Python doc strings come in a number of formats, but LSP wants markdown.
-
-    Until we can find a fast enough way of discovering and parsing each format,
-    we can do a little better by at least preserving indentation.
+def escape_plain_text(contents: str) -> str:
+    """
+    Format plain text to display nicely in environments which do not respect whitespaces.
     """
     contents = contents.replace('\t', '\u00A0' * 4)
     contents = contents.replace('  ', '\u00A0' * 2)
     return contents
 
 
+def escape_markdown(contents: str) -> str:
+    """
+    Format plain text to display nicely in Markdown environment.
+    """
+    # escape markdown syntax
+    contents = re.sub(r'([\\*_#[\]])', r'\\\1', contents)
+    # preserve white space characters
+    contents = escape_plain_text(contents)
+    return contents
+
+
+def wrap_signature(signature):
+    return '```python\n' + signature + '\n```\n'
+
+
+SERVER_SUPPORTED_MARKUP_KINDS = {'markdown', 'plaintext'}
+
+
+def choose_markup_kind(client_supported_markup_kinds: List[str]):
+    """Choose a markup kind supported by both client and the server.
+
+    This gives priority to the markup kinds provided earlier on the client preference list.
+    """
+    for kind in client_supported_markup_kinds:
+        if kind in SERVER_SUPPORTED_MARKUP_KINDS:
+            return kind
+    return 'markdown'
+
+
+def format_docstring(contents: str, markup_kind: str, signatures: Optional[List[str]] = None):
+    """Transform the provided docstring into a MarkupContent object.
+
+    If `markup_kind` is 'markdown' the docstring will get converted to
+    markdown representation using `docstring-to-markdown`; if it is
+    `plaintext`, it will be returned as plain text.
+    Call signatures of functions (or equivalent code summaries)
+    provided in optional `signatures` argument will be prepended
+    to the provided contents of the docstring if given.
+    """
+    if not isinstance(contents, str):
+        contents = ''
+
+    if markup_kind == 'markdown':
+        try:
+            value = docstring_to_markdown.convert(contents)
+            return {
+                'kind': 'markdown',
+                'value': value
+            }
+        except docstring_to_markdown.UnknownFormatError:
+            # try to escape the Markdown syntax instead:
+            value = escape_markdown(contents)
+
+        if signatures:
+            value = wrap_signature('\n'.join(signatures)) + '\n\n' + value
+
+        return {
+            'kind': 'markdown',
+            'value': value
+        }
+    value = contents
+    if signatures:
+        value = '\n'.join(signatures) + '\n\n' + value
+    return {
+        'kind': 'plaintext',
+        'value': escape_plain_text(value)
+    }
+
+
 def clip_column(column, lines, line_number):
     """
     Normalise the position as per the LSP that accepts character positions > line length

pylsp/plugins/hover.py

@@ -9,7 +9,7 @@
 
 
 @hookimpl
-def pylsp_hover(document, position):
+def pylsp_hover(config, document, position):
     code_position = _utils.position_to_jedi_linecolumn(document, position)
     definitions = document.jedi_script(use_document_path=True).infer(**code_position)
     word = document.word_at_position(position)
@@ -26,24 +26,19 @@ def pylsp_hover(document, position):
     if not definition:
         return {'contents': ''}
 
-    # raw docstring returns only doc, without signature
-    doc = _utils.format_docstring(definition.docstring(raw=True))
+    hover_capabilities = config.capabilities.get('textDocument', {}).get('hover', {})
+    supported_markup_kinds = hover_capabilities.get('contentFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
 
     # Find first exact matching signature
     signature = next((x.to_string() for x in definition.get_signatures()
                       if x.name == word), '')
 
-    contents = []
-    if signature:
-        contents.append({
-            'language': 'python',
-            'value': signature,
-        })
-
-    if doc:
-        contents.append(doc)
-
-    if not contents:
-        return {'contents': ''}
-
-    return {'contents': contents}
+    return {
+        'contents': _utils.format_docstring(
+            # raw docstring returns only doc, without signature
+            definition.docstring(raw=True),
+            preferred_markup_kind,
+            signatures=[signature] if signature else None
+        )
+    }

pylsp/plugins/jedi_completion.py

@@ -50,7 +50,10 @@ def pylsp_completions(config, document, position):
         return None
 
     completion_capabilities = config.capabilities.get('textDocument', {}).get('completion', {})
-    snippet_support = completion_capabilities.get('completionItem', {}).get('snippetSupport')
+    item_capabilities = completion_capabilities.get('completionItem', {})
+    snippet_support = item_capabilities.get('snippetSupport')
+    supported_markup_kinds = item_capabilities.get('documentationFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
 
     should_include_params = settings.get('include_params')
     should_include_class_objects = settings.get('include_class_objects', True)
@@ -69,7 +72,8 @@ def pylsp_completions(config, document, position):
     ready_completions = [
         _format_completion(
             c,
-            include_params,
+            markup_kind=preferred_markup_kind,
+            include_params=include_params,
             resolve=resolve_eagerly,
             resolve_label_or_snippet=(i < max_to_resolve)
         )
@@ -82,7 +86,8 @@ def pylsp_completions(config, document, position):
             if c.type == 'class':
                 completion_dict = _format_completion(
                     c,
-                    False,
+                    markup_kind=preferred_markup_kind,
+                    include_params=False,
                     resolve=resolve_eagerly,
                     resolve_label_or_snippet=(i < max_to_resolve)
                 )
@@ -119,12 +124,18 @@ def pylsp_completions(config, document, position):
 
 
 @hookimpl
-def pylsp_completion_item_resolve(completion_item, document):
+def pylsp_completion_item_resolve(config, completion_item, document):
     """Resolve formatted completion for given non-resolved completion"""
     shared_data = document.shared_data['LAST_JEDI_COMPLETIONS'].get(completion_item['label'])
+
+    completion_capabilities = config.capabilities.get('textDocument', {}).get('completion', {})
+    item_capabilities = completion_capabilities.get('completionItem', {})
+    supported_markup_kinds = item_capabilities.get('documentationFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
+
     if shared_data:
         completion, data = shared_data
-        return _resolve_completion(completion, data)
+        return _resolve_completion(completion, data, markup_kind=preferred_markup_kind)
     return completion_item
 
 
@@ -178,18 +189,25 @@ def use_snippets(document, position):
             not (expr_type in _ERRORS and 'import' in code))
 
 
-def _resolve_completion(completion, d):
+def _resolve_completion(completion, d, markup_kind: str):
     # pylint: disable=broad-except
     completion['detail'] = _detail(d)
     try:
-        docs = _utils.format_docstring(d.docstring())
+        docs = _utils.format_docstring(
+            d.docstring(raw=True),
+            signatures=[
+                signature.to_string()
+                for signature in d.get_signatures()
+            ],
+            markup_kind=markup_kind
+        )
     except Exception:
         docs = ''
     completion['documentation'] = docs
     return completion
 
 
-def _format_completion(d, include_params=True, resolve=False, resolve_label_or_snippet=False):
+def _format_completion(d, markup_kind: str, include_params=True, resolve=False, resolve_label_or_snippet=False):
     completion = {
         'label': _label(d, resolve_label_or_snippet),
         'kind': _TYPE_MAP.get(d.type),
@@ -198,7 +216,7 @@ def _format_completion(d, include_params=True, resolve=False, resolve_label_or_s
     }
 
     if resolve:
-        completion = _resolve_completion(completion, d)
+        completion = _resolve_completion(completion, d, markup_kind)
 
     if d.type == 'path':
         path = osp.normpath(d.name)

pylsp/plugins/rope_completion.py

@@ -4,7 +4,7 @@
 import logging
 from rope.contrib.codeassist import code_assist, sorted_proposals
 
-from pylsp import hookimpl, lsp
+from pylsp import _utils, hookimpl, lsp
 
 
 log = logging.getLogger(__name__)
@@ -16,10 +16,13 @@ def pylsp_settings():
     return {'plugins': {'rope_completion': {'enabled': False, 'eager': False}}}
 
 
-def _resolve_completion(completion, data):
+def _resolve_completion(completion, data, markup_kind):
     # pylint: disable=broad-except
     try:
-        doc = data.get_doc()
+        doc = _utils.format_docstring(
+            data.get_doc(),
+            markup_kind=markup_kind
+        )
     except Exception as e:
         log.debug("Failed to resolve Rope completion: %s", e)
         doc = ""
@@ -49,6 +52,11 @@ def pylsp_completions(config, workspace, document, position):
     rope_project = workspace._rope_project_builder(rope_config)
     document_rope = document._rope_resource(rope_config)
 
+    completion_capabilities = config.capabilities.get('textDocument', {}).get('completion', {})
+    item_capabilities = completion_capabilities.get('completionItem', {})
+    supported_markup_kinds = item_capabilities.get('documentationFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
+
     try:
         definitions = code_assist(rope_project, document.source, offset, document_rope, maxfixes=3)
     except Exception as e:  # pylint: disable=broad-except
@@ -67,7 +75,7 @@ def pylsp_completions(config, workspace, document, position):
             }
         }
         if resolve_eagerly:
-            item = _resolve_completion(item, d)
+            item = _resolve_completion(item, d, preferred_markup_kind)
         new_definitions.append(item)
 
     # most recently retrieved completion items, used for resolution
@@ -83,12 +91,18 @@ def pylsp_completions(config, workspace, document, position):
 
 
 @hookimpl
-def pylsp_completion_item_resolve(completion_item, document):
+def pylsp_completion_item_resolve(config, completion_item, document):
     """Resolve formatted completion for given non-resolved completion"""
     shared_data = document.shared_data['LAST_ROPE_COMPLETIONS'].get(completion_item['label'])
+
+    completion_capabilities = config.capabilities.get('textDocument', {}).get('completion', {})
+    item_capabilities = completion_capabilities.get('completionItem', {})
+    supported_markup_kinds = item_capabilities.get('documentationFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
+
     if shared_data:
         completion, data = shared_data
-        return _resolve_completion(completion, data)
+        return _resolve_completion(completion, data, preferred_markup_kind)
     return completion_item
 
 

pylsp/plugins/signature.py

@@ -15,28 +15,41 @@
 
 
 @hookimpl
-def pylsp_signature_help(document, position):
+def pylsp_signature_help(config, document, position):
     code_position = _utils.position_to_jedi_linecolumn(document, position)
     signatures = document.jedi_script().get_signatures(**code_position)
 
     if not signatures:
         return {'signatures': []}
 
+    signature_capabilities = config.capabilities.get('textDocument', {}).get('signatureHelp', {})
+    signature_information_support = signature_capabilities.get('signatureInformation', {})
+    supported_markup_kinds = signature_information_support.get('documentationFormat', ['markdown'])
+    preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds)
+
     s = signatures[0]
 
+    docstring = s.docstring()
+
     # Docstring contains one or more lines of signature, followed by empty line, followed by docstring
-    function_sig_lines = (s.docstring().split('\n\n') or [''])[0].splitlines()
+    function_sig_lines = (docstring.split('\n\n') or [''])[0].splitlines()
     function_sig = ' '.join([line.strip() for line in function_sig_lines])
     sig = {
         'label': function_sig,
-        'documentation': _utils.format_docstring(s.docstring(raw=True))
+        'documentation': _utils.format_docstring(
+            s.docstring(raw=True),
+            markup_kind=preferred_markup_kind
+        )
     }
 
     # If there are params, add those
     if s.params:
         sig['parameters'] = [{
             'label': p.name,
-            'documentation': _param_docs(s.docstring(), p.name)
+            'documentation': _utils.format_docstring(
+                _param_docs(docstring, p.name),
+                markup_kind=preferred_markup_kind
+            )
         } for p in s.params]
 
     # We only return a single signature because Python doesn't allow overloading

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "jedi>=0.17.2,<0.19.0",
     "python-lsp-jsonrpc>=1.0.0",
     "pluggy>=1.0.0",
+    "docstring-to-markdown",
     "ujson>=3.0.0",
     "setuptools>=39.0.0",
 ]

setup.py

@@ -5,6 +5,7 @@
 
 from setuptools import setup, find_packages
 
+
 if __name__ == "__main__":
     setup(
         name="python-lsp-server",  # to allow GitHub dependency tracking work

test/plugins/test_completion.py

@@ -160,10 +160,15 @@ def test_jedi_completion_item_resolve(config, workspace):
     assert 'detail' not in documented_hello_item
 
     resolved_documented_hello = pylsp_jedi_completion_item_resolve(
+        doc._config,
         completion_item=documented_hello_item,
         document=doc
     )
-    assert 'Sends a polite greeting' in resolved_documented_hello['documentation']
+    expected_doc = {
+        'kind': 'markdown',
+        'value': '```python\ndocumented_hello()\n```\n\n\nSends a polite greeting'
+    }
+    assert resolved_documented_hello['documentation'] == expected_doc
 
 
 def test_jedi_completion_with_fuzzy_enabled(config, workspace):
@@ -498,8 +503,8 @@ def test_jedi_completion_environment(workspace):
     completions = pylsp_jedi_completions(doc._config, doc, com_position)
     assert completions[0]['label'] == 'loghub'
 
-    resolved = pylsp_jedi_completion_item_resolve(completions[0], doc)
-    assert 'changelog generator' in resolved['documentation'].lower()
+    resolved = pylsp_jedi_completion_item_resolve(doc._config, completions[0], doc)
+    assert 'changelog generator' in resolved['documentation']['value'].lower()
 
 
 def test_document_path_completions(tmpdir, workspace_other_root_path):