Merge branch 'docstringer_improvements' into 'development'

Docstringer improvements

See merge request damask/DAMASK!738

python/damask/_orientation.py

@@ -9,31 +9,6 @@ from . import Crystal
 from . import util
 from . import tensor
 
-
-_parameter_doc = \
-       """
-        family : {'triclinic', 'monoclinic', 'orthorhombic', 'tetragonal', 'hexagonal', 'cubic'}, optional.
-            Name of the crystal family.
-            Family will be inferred if 'lattice' is given.
-        lattice : {'aP', 'mP', 'mS', 'oP', 'oS', 'oI', 'oF', 'tP', 'tI', 'hP', 'cP', 'cI', 'cF'}, optional.
-            Name of the Bravais lattice in Pearson notation.
-        a : float, optional
-            Length of lattice parameter 'a'.
-        b : float, optional
-            Length of lattice parameter 'b'.
-        c : float, optional
-            Length of lattice parameter 'c'.
-        alpha : float, optional
-            Angle between b and c lattice basis.
-        beta : float, optional
-            Angle between c and a lattice basis.
-        gamma : float, optional
-            Angle between a and b lattice basis.
-        degrees : bool, optional
-            Angles are given in degrees. Defaults to False.
-
-       """
-
 MyType = TypeVar('MyType', bound='Orientation')
 
 class Orientation(Rotation,Crystal):
@@ -93,7 +68,7 @@ class Orientation(Rotation,Crystal):
 
     """
 
-    @util.extend_docstring(extra_parameters=_parameter_doc)
+    @util.extend_docstring(adopted_parameters=Crystal.__init__)
     def __init__(self,
                  rotation: Union[FloatSequence, Rotation] = np.array([1.,0.,0.,0.]),
                  *,
@@ -267,84 +242,84 @@ class Orientation(Rotation,Crystal):
 
     @classmethod
     @util.extend_docstring(Rotation.from_random,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_random, wrapped=__init__)
     def from_random(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_quaternion,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_quaternion, wrapped=__init__)
     def from_quaternion(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_Euler_angles,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_Euler_angles, wrapped=__init__)
     def from_Euler_angles(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_axis_angle,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_axis_angle, wrapped=__init__)
     def from_axis_angle(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_basis,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_basis, wrapped=__init__)
     def from_basis(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_matrix,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_matrix, wrapped=__init__)
     def from_matrix(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_Rodrigues_vector,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_Rodrigues_vector, wrapped=__init__)
     def from_Rodrigues_vector(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_homochoric,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_homochoric, wrapped=__init__)
     def from_homochoric(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_cubochoric,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_cubochoric, wrapped=__init__)
     def from_cubochoric(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_spherical_component,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_spherical_component, wrapped=__init__)
     def from_spherical_component(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
     @classmethod
     @util.extend_docstring(Rotation.from_fiber_component,
-                           extra_parameters=_parameter_doc)
+                           adopted_parameters=Crystal.__init__)
     @util.pass_on('rotation', Rotation.from_fiber_component, wrapped=__init__)
     def from_fiber_component(cls, **kwargs) -> 'Orientation':
         return cls(**kwargs)
 
 
     @classmethod
-    @util.extend_docstring(extra_parameters=_parameter_doc)
+    @util.extend_docstring(adopted_parameters=Crystal.__init__)
     def from_directions(cls,
                         uvw: FloatSequence,
                         hkl: FloatSequence,

python/damask/util.py

@@ -8,7 +8,7 @@ import shlex as _shlex
 import re as _re
 import signal as _signal
 import fractions as _fractions
-from collections import abc as _abc
+from collections import abc as _abc, OrderedDict as _OrderedDict
 from functools import reduce as _reduce, partial as _partial, wraps as _wraps
 import inspect
 from typing import Optional as _Optional, Callable as _Callable, Union as _Union, Iterable as _Iterable, \
@@ -541,10 +541,11 @@ def shapeblender(a: _Tuple[int, ...],
 
 
 def _docstringer(docstring: _Union[str, _Callable],
-                 extra_parameters: _Optional[str] = None,
+                 adopted_parameters: _Union[None, str, _Callable] = None,
-                 # extra_examples: _Optional[str] = None,
+                 adopted_return: _Union[None, str, _Callable] = None,
-                 # extra_notes: _Optional[str] = None,
+                 adopted_notes: _Union[None, str, _Callable] = None,
-                 return_type: _Union[None, str, _Callable] = None) -> str:
+                 adopted_examples: _Union[None, str, _Callable] = None,
+                 adopted_references: _Union[None, str, _Callable] = None) -> str:
     """
     Extend a docstring.
 
@@ -552,50 +553,85 @@ def _docstringer(docstring: _Union[str, _Callable],
     ----------
     docstring : str or callable, optional
        Docstring (of callable) to extend.
-    extra_parameters : str, optional
+    adopted_* : str or callable, optional
-       Additional information to append to Parameters section.
+       Additional information to insert into/append to respective section.
-    return_type : str or callable, optional
+
-       Type of return variable.
+    Notes
+    -----
+    adopted_return fetches the typehint of a passed function instead of the docstring
 
     """
-    docstring_ = str(     docstring if isinstance(docstring,str)
+    docstring_: str = str(     docstring if isinstance(docstring,str)
-                     else docstring.__doc__ if hasattr(docstring,'__doc__')
+                          else docstring.__doc__ if callable(docstring) and docstring.__doc__
-                     else '')
+                          else '').rstrip()+'\n'
-    d = dict(Parameters=extra_parameters,
+    sections = _OrderedDict(
-             # Examples=extra_examples,
+        Parameters=adopted_parameters,
-             # Notes=extra_notes,
+        Returns=adopted_return,
-             )
+        Examples=adopted_examples,
-    for key,extra in [(k,v) for (k,v) in d.items() if v is not None]:
+        Notes=adopted_notes,
-        if not (heading := _re.search(fr'^([ ]*){key}\s*\n\1{"-"*len(key)}',
+        References=adopted_references)
-                                      docstring_,flags=_re.MULTILINE)):
-            raise RuntimeError(f"Docstring {docstring_} lacks a correctly formatted {key} section to insert values into")
-        content = [line for line in extra.split('\n') if line.strip()]
-        indent = len(heading.group(1))
-        shift = min([len(line)-len(line.lstrip(' '))-indent for line in content])
-        extra = '\n'.join([(line[shift:] if shift > 0 else
-                          f'{" "*-shift}{line}') for line in content])
-        docstring_ = _re.sub(fr'(^([ ]*){key}\s*\n\2{"-"*len(key)}[\n ]*[A-Za-z0-9_ ]*: ([^\n]+\n)*)',
-                             fr'\1{extra}\n',
-                             docstring_,flags=_re.MULTILINE)
 
-    if return_type is None:
+    for i, (key, adopted) in [(i,(k,v)) for (i,(k,v)) in enumerate(sections.items()) if v is not None]:
-        return docstring_
+        section_regex = fr'^([ ]*){key}\s*\n\1*{"-"*len(key)}\s*\n'
-    else:
+        if key=='Returns':
-        if isinstance(return_type,str):
+            if callable(adopted):
-            return_type_ = return_type
+                return_class = adopted.__annotations__.get('return','')
+                return_type_ = (_sys.modules[adopted.__module__].__name__.split('.')[0]
+                                +'.'
+                                +(return_class.__name__ if not isinstance(return_class,str) else return_class))
+            else:
+                return_type_ = adopted
+            docstring_ = _re.sub(fr'(^[ ]*{key}\s*\n\s*{"-"*len(key)}\s*\n[ ]*[A-Za-z0-9_ ]*: )(.*)\n',
+                                 fr'\1{return_type_}\n',
+                                 docstring_,flags=_re.MULTILINE)
         else:
-            return_class = return_type.__annotations__.get('return','')
+            section_content_regex = fr'{section_regex}(?P<content>.*?)\n *(\n|\Z)'
-            return_type_ = (_sys.modules[return_type.__module__].__name__.split('.')[0]
+            adopted_: str = adopted.__doc__ if callable(adopted) else adopted #type: ignore
-                            +'.'
+            try:
-                            +(return_class.__name__ if not isinstance(return_class,str) else return_class)
+                if _re.search(fr'{section_regex}', adopted_, flags=_re.MULTILINE):
-                           )
+                    adopted_ = _re.search(section_content_regex, #type: ignore
+                                          adopted_,
+                                          flags=_re.MULTILINE|_re.DOTALL).group('content')
+            except AttributeError:
+                raise RuntimeError(f"Function docstring passed for docstring section '{key}' is invalid:\n{docstring}")
+
+            docstring_indent, adopted_indent = (min([len(line)-len(line.lstrip()) for line in section.split('\n') if line.strip()])
+                                                for section in [docstring_, adopted_])
+            shift = adopted_indent - docstring_indent
+            adopted_content = '\n'.join([(line[shift:] if shift > 0 else
+                f'{" "*-shift}{line}') for line in adopted_.split('\n') if line.strip()])
+
+            if _re.search(section_regex, docstring_, flags=_re.MULTILINE):
+                docstring_section_content = _re.search(section_content_regex, # type: ignore
+                                                       docstring_,
+                                                       flags=_re.MULTILINE|_re.DOTALL).group('content')
+                a_items, d_items = (_re.findall('^[ ]*([A-Za-z0-9_ ]*?)[ ]*:',content,flags=_re.MULTILINE)
+                                    for content in [adopted_content,docstring_section_content])
+                for item in a_items:
+                    if item in d_items:
+                        adopted_content = _re.sub(fr'^([ ]*){item}.*?(?:(\n)\1([A-Za-z0-9_])|([ ]*\Z))',
+                                                  r'\1\3',
+                                                  adopted_content,
+                                                  flags=_re.MULTILINE|_re.DOTALL).rstrip(' \n')
+                docstring_ = _re.sub(fr'(^[ ]*{key}\s*\n\s*{"-"*len(key)}\s*\n.*?)\n *(\Z|\n)',
+                                     fr'\1\n{adopted_content}\n\2',
+                                     docstring_,
+                                     flags=_re.MULTILINE|_re.DOTALL)
+            else:
+                section_title = f'{" "*(shift+docstring_indent)}{key}\n{" "*(shift+docstring_indent)}{"-"*len(key)}\n'
+                section_matches = [_re.search(
+                    fr'[ ]*{list(sections.keys())[index]}\s*\n\s*{"-"*len(list(sections.keys())[index])}\s*', docstring_)
+                    for index in range(i,len(sections))]
+                subsequent_section = '\\Z' if not any(section_matches) else \
+                                        '\n'+next(item for item in section_matches if item is not None).group(0)
+                docstring_ = _re.sub(fr'({subsequent_section})',
+                                        fr'\n{section_title}{adopted_content}\n\1',
+                                        docstring_)
+    return docstring_
 
-        return _re.sub(r'(^([ ]*)Returns\s*\n\2-------\s*\n[ ]*[A-Za-z0-9_ ]*: )(.*)\n',
-                           fr'\1{return_type_}\n',
-                           docstring_,flags=_re.MULTILINE)
 
 def extend_docstring(docstring: _Union[None, str, _Callable] = None,
-                     extra_parameters: _Optional[str] = None) -> _Callable:
+                     **kwargs) -> _Callable:
     """
     Decorator: Extend the function's docstring.
 
@@ -603,8 +639,8 @@ def extend_docstring(docstring: _Union[None, str, _Callable] = None,
     ----------
     docstring : str or callable, optional
        Docstring to extend. Defaults to that of decorated function.
-    extra_parameters : str, optional
+    adopted_* : str or callable, optional
-       Additional information to append to Parameters section.
+       Additional information to insert into/append to respective section.
 
     Notes
     -----
@@ -612,10 +648,9 @@ def extend_docstring(docstring: _Union[None, str, _Callable] = None,
 
     """
     def _decorator(func):
+        if 'adopted_return' not in kwargs: kwargs['adopted_return'] = func
         func.__doc__ = _docstringer(func.__doc__ if docstring is None else docstring,
-                                    extra_parameters,
+                                    **kwargs)
-                                    func if isinstance(docstring,_Callable) else None,
-                                   )
         return func
     return _decorator
 
@@ -657,7 +692,8 @@ def pass_on(keyword: str,
         for f in [target] if wrapped is None else [target,wrapped]:
             for param in inspect.signature(f).parameters.values():
                 if      param.name != keyword \
-                    and param.name not in [p.name for p in args_]+['self','cls', 'args', 'kwargs']: args_.append(param)
+                    and param.name not in [p.name for p in args_]+['self','cls', 'args', 'kwargs']:
+                    args_.append(param.replace(kind=inspect._ParameterKind.KEYWORD_ONLY))
         wrapper.__signature__ = inspect.Signature(parameters=args_,return_annotation=inspect.signature(func).return_annotation)
         return wrapper
     return decorator

python/tests/test_util.py

@@ -230,52 +230,30 @@ class TestUtil:
     def test_Bravais_Miller_Bravais(self,vector,kw_Miller,kw_Bravais):
         assert np.all(vector == util.Miller_to_Bravais(**{kw_Miller:util.Bravais_to_Miller(**{kw_Bravais:vector})}))
 
-
+    @pytest.mark.parametrize('adopted_parameters',[
-    @pytest.mark.parametrize('extra_parameters',["""
+            pytest.param("""
             p2 : str, optional
                 p2 description 1
                 p2 description 2
             """,
-            """
+            id = 'standard'),
+            pytest.param("""
 
                         p2 : str, optional
                             p2 description 1
                             p2 description 2
 
             """,
-            """
+            id = 'indented'),
+            pytest.param("""
 p2 : str, optional
     p2 description 1
     p2 description 2
-            """])
+            """,
-    @pytest.mark.parametrize('invalid_docstring',["""
+            id = 'no_indent')])
-                        Function description
+    def test_extend_docstring_parameters_string(self,adopted_parameters):
-
-                        Parameters ----------
-                        p0 : numpy.ndarray, shape (...,4)
-                            p0 description 1
-                            p0 description 2
-                        p1 : int, optional
-                            p1 description
-
-                        Remaining description
-                    """,
-                    """
-                        Function description
-
-                        Parameters
-                         ----------
-                        p0 : numpy.ndarray, shape (...,4)
-                            p0 description 1
-                            p0 description 2
-                        p1 : int, optional
-                            p1 description
-
-                        Remaining description
-                    """,])
-    def test_extend_docstring_parameters(self,extra_parameters,invalid_docstring):
         test_docstring = """
-            Function description
+            Function description.
 
             Parameters
             ----------
@@ -285,10 +263,10 @@ p2 : str, optional
             p1 : int, optional
                 p1 description
 
-            Remaining description
+            Remaining description\n"""
+        assert util._docstringer(test_docstring,adopted_parameters) ==\
         """
-        expected = """
+            Function description.
-            Function description
 
             Parameters
             ----------
@@ -301,47 +279,221 @@ p2 : str, optional
                 p2 description 1
                 p2 description 2
 
-            Remaining description
+            Remaining description\n"""
-        """.split("\n")
-        assert expected == util._docstringer(test_docstring,extra_parameters).split('\n')
-        with pytest.raises(RuntimeError):
-            util._docstringer(invalid_docstring,extra_parameters)
 
-    def test_replace_docstring_return_type(self):
+    def test_extend_docstring_parameters_function(self):
+        test_docstring = """
+            Function description.
+
+            Parameters
+            ----------
+            p0 : numpy.ndarray, shape (...,4)
+                p0 description 1
+
+        """
+
+        def testfunction_1():
+            """
+            Function description.
+
+            Parameters
+            ----------
+            p1 : int, optional
+                p1 description
+
+            Notes
+            -----
+            Function Notes 1
+            Function Notes 2
+
+            References
+            ----------
+            Reference 1
+            <reference link>
+            Reference 2
+            <reference link>
+
+            """
+            pass
+
+        assert (test_docstring:=util._docstringer(test_docstring,adopted_references = testfunction_1)) == \
+        """
+            Function description.
+
+            Parameters
+            ----------
+            p0 : numpy.ndarray, shape (...,4)
+                p0 description 1
+
+            References
+            ----------
+            Reference 1
+            <reference link>
+            Reference 2
+            <reference link>\n"""
+        assert (test_docstring:=util._docstringer(test_docstring,adopted_notes = testfunction_1)) == \
+        """
+            Function description.
+
+            Parameters
+            ----------
+            p0 : numpy.ndarray, shape (...,4)
+                p0 description 1
+
+            Notes
+            -----
+            Function Notes 1
+            Function Notes 2
+
+            References
+            ----------
+            Reference 1
+            <reference link>
+            Reference 2
+            <reference link>\n"""
+
+        def testfunction_2():
+            """
+            Function description.
+
+            References
+            ----------
+            Reference 3
+            <reference link>
+
+            """
+
+        assert (test_docstring:=util._docstringer(test_docstring,adopted_references = testfunction_2)) == \
+        """
+            Function description.
+
+            Parameters
+            ----------
+            p0 : numpy.ndarray, shape (...,4)
+                p0 description 1
+
+            Notes
+            -----
+            Function Notes 1
+            Function Notes 2
+
+            References
+            ----------
+            Reference 1
+            <reference link>
+            Reference 2
+            <reference link>
+            Reference 3
+            <reference link>\n"""
+
+    def return_bound_method():
+        class TestClassDecorated:
+            def decorated_func_bound(self) -> 'TestClassDecorated':
+                pass
+        return TestClassDecorated.decorated_func_bound
+
+    def return_simple_function():
+        class TestClassDecorated:
+            pass
+
+        def decorated_func() -> TestClassDecorated:
+            pass
+        return decorated_func
+
+    @pytest.mark.parametrize('adopted_return',[
+            pytest.param(return_simple_function(),
+            id = 'decorated_func'),
+            pytest.param(return_bound_method(),
+            id = 'decorated_func_bound'),
+            pytest.param('test_util.TestClassDecorated',
+            id = 'decorated_func_bound')])
+    def test_replace_docstring_return(self,adopted_return):
         class TestClassOriginal:
             pass
 
         def original_func() -> TestClassOriginal:
             pass
 
-        class TestClassDecorated:
-            def decorated_func_bound(self) -> 'TestClassDecorated':
-                pass
-
-        def decorated_func() -> TestClassDecorated:
-            pass
-
         original_func.__doc__ = """
-            Function description/Parameters
+            Function description.
 
             Returns
             -------
             Return value : test_util.TestClassOriginal
 
             Remaining description
-        """
+            """
 
-        expected = """
+
-            Function description/Parameters
+        assert util._docstringer(original_func,adopted_return=adopted_return) == """
+            Function description.
 
             Returns
             -------
             Return value : test_util.TestClassDecorated
 
-            Remaining description
+            Remaining description\n"""
-        """
+
-        assert expected == util._docstringer(original_func,return_type=decorated_func)
+
-        assert expected == util._docstringer(original_func,return_type=TestClassDecorated.decorated_func_bound)
+    @pytest.mark.parametrize('adopted_func_doc',[
+            pytest.param("""
+            Function description.
+
+            Parameters
+            ----------
+            b : float 4
+                b description
+            c : float 5
+                c description differing
+            d : float 6
+                d description
+
+            Remaining description\n
+            """,
+            id = 'append'),
+            pytest.param("""
+            Function description.
+
+            Parameters
+            ----------
+            d : float 6
+                d description
+            a : float 7
+                a description\n""",
+            id = 'insert')])
+    def test_extend_docstring_overlapping_section_content(self,adopted_func_doc):
+
+        original_func_doc = """
+            Function description.
+
+            Parameters
+            ----------
+            a : float 1
+                a description
+            b : float 2
+                b description
+            c : float 3
+                c description
+
+            Remaining description\n"""
+
+        expected = """
+            Function description.
+
+            Parameters
+            ----------
+            a : float 1
+                a description
+            b : float 2
+                b description
+            c : float 3
+                c description
+            d : float 6
+                d description
+
+            Remaining description\n"""
+
+        assert util._docstringer(original_func_doc,adopted_parameters=adopted_func_doc) == expected
 
     def test_passon_result(self):
         def testfunction_inner(a=None,b=None):
@@ -377,4 +529,4 @@ p2 : str, optional
             return kwargs['inner_result']+kwargs['c']+kwargs['d']
 
         assert pydoc.render_doc(testfunction_outer, renderer=pydoc.plaintext).split("\n")[-2] ==\
-              'testfunction_outer(a=None, b=None, *, c=None, d=None) -> int'
+              'testfunction_outer(*, a=None, b=None, c=None, d=None) -> int'