├── examples ├── some_package │ ├── __init__.py │ └── some_module.py ├── class_decorators.py ├── type_checking.py ├── old_enum.py ├── tree.py ├── performance │ ├── cpu_use.py │ ├── mem_use.py │ └── generate_heavy_modules.py ├── evaluate.py └── stringify.py ├── README.rst └── pep-0563.rst /examples/some_package/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/class_decorators.py: -------------------------------------------------------------------------------- 1 | from typing import List, get_type_hints 2 | 3 | 4 | def class_decorator(cls): 5 | annotations = get_type_hints(cls) 6 | print(f'Annotations for {cls}: {annotations}') 7 | return cls 8 | 9 | 10 | @class_decorator 11 | class C: 12 | singleton: 'C' = None 13 | -------------------------------------------------------------------------------- /examples/type_checking.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, List, get_type_hints 2 | 3 | if TYPE_CHECKING: 4 | from stringify import C 5 | 6 | 7 | def function(arg: 'List[C]') -> None: 8 | ... 9 | 10 | 11 | def main(): 12 | annotations = get_type_hints(function) # this raises NameError on 'C' 13 | print(f'Annotations for {function}: {annotations}') 14 | 15 | 16 | if __name__ == '__main__': 17 | main() 18 | -------------------------------------------------------------------------------- /examples/old_enum.py: -------------------------------------------------------------------------------- 1 | """This example will no longer work after PEP 5XX is accepted and implemented.""" 2 | 3 | from enum import Enum 4 | from typing import List, get_type_hints 5 | 6 | 7 | def class_decorator(cls): 8 | annotations = get_type_hints(cls) 9 | print(f'Annotations for {cls}: {annotations}') 10 | return cls 11 | 12 | 13 | @class_decorator 14 | class Restaurant: 15 | class MenuOption(Enum): 16 | SPAM = 1 17 | EGGS = 2 18 | 19 | default_menu: List[MenuOption] = [] 20 | -------------------------------------------------------------------------------- /examples/some_package/some_module.py: -------------------------------------------------------------------------------- 1 | import stringify 2 | 3 | 4 | class D: 5 | D_CLASS_VAR = 'D_CLASS_VAR_VALUE' 6 | 7 | class E: 8 | E_CLASS_VAR: 'stringify.SOME_GLOBAL + D.D_CLASS_VAR' = 'E_CLASS_VAR_VALUE' 9 | 10 | def method_with_arbitrary_annotations(a: 'stringify.SOME_GLOBAL + "3"') -> '{"s": D.E.E_CLASS_VAR, "t": D.D_CLASS_VAR}': 11 | """Note: class variables need fully-qualified prefixes now. 12 | 13 | Also, the no_type_check decorator here is missing to demonstrate that 14 | postponed evaluation doesn't require it. It's still recommended though. 15 | """ 16 | -------------------------------------------------------------------------------- /examples/tree.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from typing import get_type_hints 3 | 4 | 5 | def resolve_types(cls): 6 | globalns = sys.modules[cls.__module__].__dict__ 7 | return get_type_hints(cls, globalns=globalns) 8 | 9 | 10 | def class_decorator(cls): 11 | annotations = resolve_types(cls) 12 | print(annotations) 13 | return cls 14 | 15 | 16 | # a class decorator won't work because the name "Tree" isn't assigned yet 17 | #@class_decorator 18 | class Tree: 19 | left: 'Tree' 20 | right: 'Tree' 21 | 22 | def __init__(self, left: 'Tree', right: 'Tree'): 23 | self.left = left 24 | self.right = right 25 | 26 | 27 | print(resolve_types(Tree)) 28 | -------------------------------------------------------------------------------- /examples/performance/cpu_use.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | 4 | 5 | t0 = time.time() 6 | try: 7 | import heavy_module_no_annotations 8 | except ImportError: 9 | print('Run `generate_heavy_modules.py` first.', file=sys.stderr) 10 | sys.exit(1) 11 | t1 = time.time() 12 | no_ann = t1 - t0 13 | print(f"No Annotations: {no_ann:.2f}s") 14 | 15 | t0 = time.time() 16 | try: 17 | import heavy_module_annotations 18 | except ImportError: 19 | print('Run `generate_heavy_modules.py` first.', file=sys.stderr) 20 | sys.exit(1) 21 | t1 = time.time() 22 | with_ann = t1 - t0 23 | print(f"Annotations: {with_ann:.2f}s") 24 | 25 | print(f"Difference: {with_ann - no_ann:.2f} seconds") 26 | -------------------------------------------------------------------------------- /examples/performance/mem_use.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import tracemalloc 3 | 4 | 5 | tracemalloc.start() 6 | try: 7 | import heavy_module_no_annotations 8 | except ImportError: 9 | print('Run `generate_heavy_modules.py` first.', file=sys.stderr) 10 | sys.exit(1) 11 | snapshot = tracemalloc.take_snapshot() 12 | mem_size2 = sum(stat.size for stat in snapshot.statistics('filename')) 13 | print(f"No Annotations: {mem_size2/1024/1024:.2f} MB") 14 | tracemalloc.stop() 15 | 16 | tracemalloc.start() 17 | try: 18 | import heavy_module_annotations 19 | except ImportError: 20 | print('Run `generate_heavy_modules.py` first.', file=sys.stderr) 21 | sys.exit(1) 22 | snapshot = tracemalloc.take_snapshot() 23 | mem_size1 = sum(stat.size for stat in snapshot.statistics('filename')) 24 | print(f"Annotations: {mem_size1/1024/1024:.2f} MB") 25 | tracemalloc.stop() 26 | 27 | print(f"Difference: {(mem_size1-mem_size2)/1024/1024:.2f} MB") 28 | -------------------------------------------------------------------------------- /examples/performance/generate_heavy_modules.py: -------------------------------------------------------------------------------- 1 | with open('heavy_module_annotations.py', 'w') as f: 2 | f.write("from typing import Any, Dict, Iterator, Mapping, Optional, List, TypeVar, Union\n") 3 | f.write("from configparser import ConfigParser\n") 4 | for i in range(2000): 5 | f.write(f"def f{i}(a: Dict[str, List[Optional[str]]], b: Optional[List[int]]) -> Union[str, bytes]: ...\n") 6 | f.write(f"def g{i}(a: Optional[ConfigParser], b: Mapping[int, int] = {{}}) -> Any: ...\n") 7 | f.write(f"def h{i}(a: Any, *args: int) -> Optional[Dict[int, Any]]: ...\n") 8 | f.write(f"_T{i} = TypeVar('T{i}', bound='C{i}')\n") 9 | f.write(f"""class C{i}(Iterator[str]): 10 | field1: 'C{i}' 11 | field2: Mapping[bytes, str] 12 | def meth{i}(self: _T{i}, b: int) -> Optional[Dict[str, _T{i}]]: 13 | ...\n""") 14 | f.write(f"def i{i}(a: C{i}, b: str = None) -> str: ...\n") 15 | 16 | with open('heavy_module_no_annotations.py', 'w') as f: 17 | f.write("from configparser import ConfigParser\n") 18 | for i in range(200): 19 | f.write(f"def f{i}(a, b): ...\n") 20 | f.write(f"def g{i}(a, b = {{}}): ...\n") 21 | f.write(f"def h{i}(a, *args): ...\n") 22 | f.write(f"""class C{i}: 23 | def meth{i}(self, b): 24 | ...\n""") 25 | f.write(f"def i{i}(a, b = None): ...\n") 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | PEP 563: Postponed Evaluation of Annotations 3 | ============================================ 4 | 5 | This GitHub repo is used for drafting of this PEP. Technical discussion 6 | around the PEP should happen in the issue tracker. The official PEPS 7 | repository is only updated when a new draft is posted on python-dev. 8 | The python.org site automatically updates (with a slight delay, 9 | typically in the order of 5-60 minutes) whenever the python/peps repo is 10 | updated. 11 | 12 | 13 | Authors 14 | ------- 15 | 16 | * Łukasz Langa 17 | 18 | 19 | Important dates 20 | --------------- 21 | 22 | * May 24, 2015: Python 3.5.0 beta 1 -- PEP 484 accepted, ``typing`` 23 | checked into the CPython repo 24 | 25 | * September 13, 2015: Python 3.5.0 final release; ``typing`` is 26 | available on a provisional basis 27 | 28 | * September 7, 2016: PEP 526 accepted on a provisional basis 29 | 30 | * December 23, 2016: Python 3.6.0 final release; ``typing`` stays 31 | provisional for the course of the 3.6 releases 32 | 33 | * January 29, 2018: Python 3.7.0 beta 1, feature freeze for the release, 34 | incl. the ``typing`` module, and ``from __future__ import annotations`` 35 | 36 | * June 15, 2018: Python 3.7.0 final release, the ``typing`` module is no 37 | longer provisional, variable annotations are no longer provisional. 38 | 39 | The dates for Python 3.7 are based on PEP 537 and may still change. 40 | 41 | 42 | Important URLs 43 | -------------- 44 | 45 | The python.org rendering of PEP 563 will live at 46 | https://www.python.org/dev/peps/pep-0563/. 47 | 48 | Related PEPs: 49 | 50 | * Type Hints: https://www.python.org/dev/peps/pep-0484/. 51 | 52 | * Function annotations: https://www.python.org/dev/peps/pep-3107/. 53 | 54 | * Variable annotations: https://www.python.org/dev/peps/pep-0526/. 55 | -------------------------------------------------------------------------------- /examples/evaluate.py: -------------------------------------------------------------------------------- 1 | """This is an example of postponed evaluation from a different module. It also 2 | includes a tricky example with nested classes.""" 3 | 4 | import sys 5 | 6 | import stringify 7 | from some_package.some_module import D 8 | 9 | 10 | def evaluate_other_module(): 11 | """Demonstrates how to evaluate annotations for a different module.""" 12 | stringify_globals = stringify.__dict__ 13 | mod_annotations = {} 14 | for k, v in stringify.__annotations__.items(): 15 | mod_annotations[k] = eval(v, stringify_globals, stringify_globals) 16 | print('Other module:', mod_annotations) 17 | 18 | 19 | def evaluate_method(C): 20 | """Demonstrates how to evaluate method annotations.""" 21 | 22 | # unbound, just a function object 23 | c_meth = C.method_with_arbitrary_annotations 24 | c_meth_globals = c_meth.__globals__ 25 | c_meth_annotations = {} 26 | for k, v in C.method_with_arbitrary_annotations.__annotations__.items(): 27 | c_meth_annotations[k] = eval(v, c_meth_globals, c_meth_globals) 28 | print(f'Method on {C}: {c_meth_annotations}') 29 | 30 | # bound, can use __self__ 31 | c_meth = C().method_with_arbitrary_annotations 32 | c_meth_globals = c_meth.__globals__ 33 | c_meth_annotations = {} 34 | for k, v in C.method_with_arbitrary_annotations.__annotations__.items(): 35 | c_meth_annotations[k] = eval(v, c_meth_globals, c_meth_globals) 36 | print(f'Method on an instance of {C}: {c_meth_annotations}') 37 | 38 | 39 | def evaluate_classvar(C): 40 | """Demonstrates how to evaluate classvar annotations.""" 41 | 42 | c_globals = sys.modules[C.__module__].__dict__ 43 | c_annotations = {} 44 | for k, v in C.__annotations__.items(): 45 | c_annotations[k] = eval(v, c_globals, c_globals) 46 | print(f'Class vars for {C}: {c_annotations}') 47 | 48 | 49 | if __name__ == '__main__': 50 | evaluate_other_module() 51 | evaluate_method(C=stringify.C) 52 | evaluate_method(C=D.E) 53 | evaluate_classvar(C=stringify.C) 54 | evaluate_classvar(C=D.E) 55 | -------------------------------------------------------------------------------- /examples/stringify.py: -------------------------------------------------------------------------------- 1 | """A bunch of examples of how to resolve annotations whose evaluation was 2 | postponed. 3 | 4 | Note: globals and functions are purposefully out of order to demonstrate the 5 | forward reference problem. 6 | """ 7 | 8 | import sys 9 | from typing import no_type_check 10 | 11 | 12 | @no_type_check 13 | def function_with_arbitrary_annotations(a: 'SOME_GLOBAL + "1"') -> '{"s": SOME_GLOBAL}': 14 | """Note: the no_type_check decorator is not mandatory but is recommended.""" 15 | 16 | 17 | class C: 18 | CLASS_VAR: 'SOME_GLOBAL + "2"' = 'CLASS_VAR_VALUE' 19 | 20 | def method_with_arbitrary_annotations(a: 'SOME_GLOBAL + "3"') -> '{"s": C.CLASS_VAR}': 21 | """Note: CLASS_VAR needs to be prefixed with the class it comes from. 22 | Only module-level names can be used for annotations now. 23 | 24 | Also, the no_type_check decorator here is missing to demonstrate that 25 | postponed evaluation doesn't require it. It's still recommended though. 26 | """ 27 | 28 | 29 | def evaluate_current_module(): 30 | """Demonstrates how to evaluate annotations for the current module.""" 31 | mod_annotations = {} 32 | for k, v in __annotations__.items(): 33 | mod_annotations[k] = eval(v, globals(), globals()) 34 | print('Current module:', mod_annotations) 35 | 36 | 37 | def evaluate_method(): 38 | """Demonstrates how to evaluate method annotations.""" 39 | 40 | # unbound, just a function object 41 | c_meth = C.method_with_arbitrary_annotations 42 | c_meth_globals = c_meth.__globals__ 43 | c_meth_annotations = {} 44 | for k, v in C.method_with_arbitrary_annotations.__annotations__.items(): 45 | c_meth_annotations[k] = eval(v, c_meth_globals, c_meth_globals) 46 | print(f'Method on {C}: {c_meth_annotations}') 47 | 48 | # bound, can use __self__ 49 | c_meth = C().method_with_arbitrary_annotations 50 | c_meth_globals = c_meth.__globals__ 51 | c_meth_annotations = {} 52 | for k, v in C.method_with_arbitrary_annotations.__annotations__.items(): 53 | c_meth_annotations[k] = eval(v, c_meth_globals, c_meth_globals) 54 | print(f'Method on an instance of {C}: {c_meth_annotations}') 55 | 56 | 57 | def evaluate_classvar(): 58 | """Demonstrates how to evaluate classvar annotations.""" 59 | c_globals = sys.modules[C.__module__].__dict__ 60 | c_annotations = {} 61 | for k, v in C.__annotations__.items(): 62 | c_annotations[k] = eval(v, c_globals, c_globals) 63 | print(f'Class vars for {C}: {c_annotations}') 64 | 65 | 66 | SOME_GLOBAL: 'OTHER_GLOBAL' = 'SOME_GLOBAL_VALUE' 67 | OTHER_GLOBAL: 'OTHER_GLOBAL' = 'OTHER_GLOBAL_VALUE' 68 | 69 | 70 | if __name__ == '__main__': 71 | evaluate_current_module() 72 | evaluate_method() 73 | evaluate_classvar() 74 | -------------------------------------------------------------------------------- /pep-0563.rst: -------------------------------------------------------------------------------- 1 | PEP: 563 2 | Title: Postponed Evaluation of Annotations 3 | Version: $Revision$ 4 | Last-Modified: $Date$ 5 | Author: Łukasz Langa 6 | Discussions-To: Python-Dev 7 | Status: Accepted 8 | Type: Standards Track 9 | Content-Type: text/x-rst 10 | Created: 8-Sep-2017 11 | Python-Version: 3.7 12 | Post-History: 1-Nov-2017, 21-Nov-2017 13 | Resolution: https://mail.python.org/pipermail/python-dev/2017-December/151042.html 14 | 15 | 16 | Abstract 17 | ======== 18 | 19 | PEP 3107 introduced syntax for function annotations, but the semantics 20 | were deliberately left undefined. PEP 484 introduced a standard meaning 21 | to annotations: type hints. PEP 526 defined variable annotations, 22 | explicitly tying them with the type hinting use case. 23 | 24 | This PEP proposes changing function annotations and variable annotations 25 | so that they are no longer evaluated at function definition time. 26 | Instead, they are preserved in ``__annotations__`` in string form. 27 | 28 | This change is going to be introduced gradually, starting with a new 29 | ``__future__`` import in Python 3.7. 30 | 31 | 32 | Rationale and Goals 33 | =================== 34 | 35 | PEP 3107 added support for arbitrary annotations on parts of a function 36 | definition. Just like default values, annotations are evaluated at 37 | function definition time. This creates a number of issues for the type 38 | hinting use case: 39 | 40 | * forward references: when a type hint contains names that have not been 41 | defined yet, that definition needs to be expressed as a string 42 | literal; 43 | 44 | * type hints are executed at module import time, which is not 45 | computationally free. 46 | 47 | Postponing the evaluation of annotations solves both problems. 48 | 49 | Non-goals 50 | --------- 51 | 52 | Just like in PEP 484 and PEP 526, it should be emphasized that **Python 53 | will remain a dynamically typed language, and the authors have no desire 54 | to ever make type hints mandatory, even by convention.** 55 | 56 | This PEP is meant to solve the problem of forward references in type 57 | annotations. There are still cases outside of annotations where 58 | forward references will require usage of string literals. Those are 59 | listed in a later section of this document. 60 | 61 | Annotations without forced evaluation enable opportunities to improve 62 | the syntax of type hints. This idea will require its own separate PEP 63 | and is not discussed further in this document. 64 | 65 | Non-typing usage of annotations 66 | ------------------------------- 67 | 68 | While annotations are still available for arbitrary use besides type 69 | checking, it is worth mentioning that the design of this PEP, as well 70 | as its precursors (PEP 484 and PEP 526), is predominantly motivated by 71 | the type hinting use case. 72 | 73 | In Python 3.8 PEP 484 will graduate from provisional status. Other 74 | enhancements to the Python programming language like PEP 544, PEP 557, 75 | or PEP 560, are already being built on this basis as they depend on 76 | type annotations and the ``typing`` module as defined by PEP 484. 77 | In fact, the reason PEP 484 is staying provisional in Python 3.7 is to 78 | enable rapid evolution for another release cycle that some of the 79 | aforementioned enhancements require. 80 | 81 | With this in mind, uses for annotations incompatible with the 82 | aforementioned PEPs should be considered deprecated. 83 | 84 | 85 | Implementation 86 | ============== 87 | 88 | In Python 4.0, function and variable annotations will no longer be 89 | evaluated at definition time. Instead, a string form will be preserved 90 | in the respective ``__annotations__`` dictionary. Static type checkers 91 | will see no difference in behavior, whereas tools using annotations at 92 | runtime will have to perform postponed evaluation. 93 | 94 | The string form is obtained from the AST during the compilation step, 95 | which means that the string form might not preserve the exact formatting 96 | of the source. Note: if an annotation was a string literal already, it 97 | will still be wrapped in a string. 98 | 99 | Annotations need to be syntactically valid Python expressions, also when 100 | passed as literal strings (i.e. ``compile(literal, '', 'eval')``). 101 | Annotations can only use names present in the module scope as postponed 102 | evaluation using local names is not reliable (with the sole exception of 103 | class-level names resolved by ``typing.get_type_hints()``). 104 | 105 | Note that as per PEP 526, local variable annotations are not evaluated 106 | at all since they are not accessible outside of the function's closure. 107 | 108 | Enabling the future behavior in Python 3.7 109 | ------------------------------------------ 110 | 111 | The functionality described above can be enabled starting from Python 112 | 3.7 using the following special import:: 113 | 114 | from __future__ import annotations 115 | 116 | A reference implementation of this functionality is available 117 | `on GitHub `_. 118 | 119 | 120 | Resolving Type Hints at Runtime 121 | =============================== 122 | 123 | To resolve an annotation at runtime from its string form to the result 124 | of the enclosed expression, user code needs to evaluate the string. 125 | 126 | For code that uses type hints, the 127 | ``typing.get_type_hints(obj, globalns=None, localns=None)`` function 128 | correctly evaluates expressions back from its string form. Note that 129 | all valid code currently using ``__annotations__`` should already be 130 | doing that since a type annotation can be expressed as a string literal. 131 | 132 | For code which uses annotations for other purposes, a regular 133 | ``eval(ann, globals, locals)`` call is enough to resolve the 134 | annotation. 135 | 136 | In both cases it's important to consider how globals and locals affect 137 | the postponed evaluation. An annotation is no longer evaluated at the 138 | time of definition and, more importantly, *in the same scope* where it 139 | was defined. Consequently, using local state in annotations is no 140 | longer possible in general. As for globals, the module where the 141 | annotation was defined is the correct context for postponed evaluation. 142 | 143 | The ``get_type_hints()`` function automatically resolves the correct 144 | value of ``globalns`` for functions and classes. It also automatically 145 | provides the correct ``localns`` for classes. 146 | 147 | When running ``eval()``, 148 | the value of globals can be gathered in the following way: 149 | 150 | * function objects hold a reference to their respective globals in an 151 | attribute called ``__globals__``; 152 | 153 | * classes hold the name of the module they were defined in, this can be 154 | used to retrieve the respective globals:: 155 | 156 | cls_globals = vars(sys.modules[SomeClass.__module__]) 157 | 158 | Note that this needs to be repeated for base classes to evaluate all 159 | ``__annotations__``. 160 | 161 | * modules should use their own ``__dict__``. 162 | 163 | The value of ``localns`` cannot be reliably retrieved for functions 164 | because in all likelihood the stack frame at the time of the call no 165 | longer exists. 166 | 167 | For classes, ``localns`` can be composed by chaining vars of the given 168 | class and its base classes (in the method resolution order). Since slots 169 | can only be filled after the class was defined, we don't need to consult 170 | them for this purpose. 171 | 172 | Runtime annotation resolution and class decorators 173 | -------------------------------------------------- 174 | 175 | Metaclasses and class decorators that need to resolve annotations for 176 | the current class will fail for annotations that use the name of the 177 | current class. Example:: 178 | 179 | def class_decorator(cls): 180 | annotations = get_type_hints(cls) # raises NameError on 'C' 181 | print(f'Annotations for {cls}: {annotations}') 182 | return cls 183 | 184 | @class_decorator 185 | class C: 186 | singleton: 'C' = None 187 | 188 | This was already true before this PEP. The class decorator acts on 189 | the class before it's assigned a name in the current definition scope. 190 | 191 | Runtime annotation resolution and ``TYPE_CHECKING`` 192 | --------------------------------------------------- 193 | 194 | Sometimes there's code that must be seen by a type checker but should 195 | not be executed. For such situations the ``typing`` module defines a 196 | constant, ``TYPE_CHECKING``, that is considered ``True`` during type 197 | checking but ``False`` at runtime. Example:: 198 | 199 | import typing 200 | 201 | if typing.TYPE_CHECKING: 202 | import expensive_mod 203 | 204 | def a_func(arg: expensive_mod.SomeClass) -> None: 205 | a_var: expensive_mod.SomeClass = arg 206 | ... 207 | 208 | This approach is also useful when handling import cycles. 209 | 210 | Trying to resolve annotations of ``a_func`` at runtime using 211 | ``typing.get_type_hints()`` will fail since the name ``expensive_mod`` 212 | is not defined (``TYPE_CHECKING`` variable being ``False`` at runtime). 213 | This was already true before this PEP. 214 | 215 | 216 | Backwards Compatibility 217 | ======================= 218 | 219 | This is a backwards incompatible change. Applications depending on 220 | arbitrary objects to be directly present in annotations will break 221 | if they are not using ``typing.get_type_hints()`` or ``eval()``. 222 | 223 | Annotations that depend on locals at the time of the function 224 | definition will not be resolvable later. Example:: 225 | 226 | def generate(): 227 | A = Optional[int] 228 | class C: 229 | field: A = 1 230 | def method(self, arg: A) -> None: ... 231 | return C 232 | X = generate() 233 | 234 | Trying to resolve annotations of ``X`` later by using 235 | ``get_type_hints(X)`` will fail because ``A`` and its enclosing scope no 236 | longer exists. Python will make no attempt to disallow such annotations 237 | since they can often still be successfully statically analyzed, which is 238 | the predominant use case for annotations. 239 | 240 | Annotations using nested classes and their respective state are still 241 | valid. They can use local names or the fully qualified name. Example:: 242 | 243 | class C: 244 | field = 'c_field' 245 | def method(self) -> C.field: # this is OK 246 | ... 247 | 248 | def method(self) -> field: # this is OK 249 | ... 250 | 251 | def method(self) -> C.D: # this is OK 252 | ... 253 | 254 | def method(self) -> D: # this is OK 255 | ... 256 | 257 | class D: 258 | field2 = 'd_field' 259 | def method(self) -> C.D.field2: # this is OK 260 | ... 261 | 262 | def method(self) -> D.field2: # this is OK 263 | ... 264 | 265 | def method(self) -> field2: # this is OK 266 | ... 267 | 268 | def method(self) -> field: # this FAILS, class D doesn't 269 | ... # see C's attributes, This was 270 | # already true before this PEP. 271 | 272 | In the presence of an annotation that isn't a syntactically valid 273 | expression, SyntaxError is raised at compile time. However, since names 274 | aren't resolved at that time, no attempt is made to validate whether 275 | used names are correct or not. 276 | 277 | Deprecation policy 278 | ------------------ 279 | 280 | Starting with Python 3.7, a ``__future__`` import is required to use the 281 | described functionality. No warnings are raised. 282 | 283 | In Python 3.8 a ``PendingDeprecationWarning`` is raised by the 284 | compiler in the presence of type annotations in modules without the 285 | ``__future__`` import. 286 | 287 | Starting with Python 3.9 the warning becomes a ``DeprecationWarning``. 288 | 289 | In Python 4.0 this will become the default behavior. Use of annotations 290 | incompatible with this PEP is no longer supported. 291 | 292 | 293 | Forward References 294 | ================== 295 | 296 | Deliberately using a name before it was defined in the module is called 297 | a forward reference. For the purpose of this section, we'll call 298 | any name imported or defined within a ``if TYPE_CHECKING:`` block 299 | a forward reference, too. 300 | 301 | This PEP addresses the issue of forward references in *type annotations*. 302 | The use of string literals will no longer be required in this case. 303 | However, there are APIs in the ``typing`` module that use other syntactic 304 | constructs of the language, and those will still require working around 305 | forward references with string literals. The list includes: 306 | 307 | * type definitions:: 308 | 309 | T = TypeVar('T', bound='') 310 | UserId = NewType('UserId', '') 311 | Employee = NamedTuple('Employee', [('name', '', ('id', '')]) 312 | 313 | * aliases:: 314 | 315 | Alias = Optional[''] 316 | AnotherAlias = Union['', ''] 317 | YetAnotherAlias = '' 318 | 319 | * casting:: 320 | 321 | cast('', value) 322 | 323 | * base classes:: 324 | 325 | class C(Tuple['', '']): ... 326 | 327 | Depending on the specific case, some of the cases listed above might be 328 | worked around by placing the usage in a ``if TYPE_CHECKING:`` block. 329 | This will not work for any code that needs to be available at runtime, 330 | notably for base classes and casting. For named tuples, using the new 331 | class definition syntax introduced in Python 3.6 solves the issue. 332 | 333 | In general, fixing the issue for *all* forward references requires 334 | changing how module instantiation is performed in Python, from the 335 | current single-pass top-down model. This would be a major change in the 336 | language and is out of scope for this PEP. 337 | 338 | 339 | Rejected Ideas 340 | ============== 341 | 342 | Keeping the ability to use function local state when defining annotations 343 | ------------------------------------------------------------------------- 344 | 345 | With postponed evaluation, this would require keeping a reference to 346 | the frame in which an annotation got created. This could be achieved 347 | for example by storing all annotations as lambdas instead of strings. 348 | 349 | This would be prohibitively expensive for highly annotated code as the 350 | frames would keep all their objects alive. That includes predominantly 351 | objects that won't ever be accessed again. 352 | 353 | To be able to address class-level scope, the lambda approach would 354 | require a new kind of cell in the interpreter. This would proliferate 355 | the number of types that can appear in ``__annotations__``, as well as 356 | wouldn't be as introspectable as strings. 357 | 358 | Note that in the case of nested classes, the functionality to get the 359 | effective "globals" and "locals" at definition time is provided by 360 | ``typing.get_type_hints()``. 361 | 362 | If a function generates a class or a function with annotations that 363 | have to use local variables, it can populate the given generated 364 | object's ``__annotations__`` dictionary directly, without relying on 365 | the compiler. 366 | 367 | Disallowing local state usage for classes, too 368 | ---------------------------------------------- 369 | 370 | This PEP originally proposed limiting names within annotations to only 371 | allow names from the model-level scope, including for classes. The 372 | author argued this makes name resolution unambiguous, including in cases 373 | of conflicts between local names and module-level names. 374 | 375 | This idea was ultimately rejected in case of classes. Instead, 376 | ``typing.get_type_hints()`` got modified to populate the local namespace 377 | correctly if class-level annotations are needed. 378 | 379 | The reasons for rejecting the idea were that it goes against the 380 | intuition of how scoping works in Python, and would break enough 381 | existing type annotations to make the transition cumbersome. Finally, 382 | local scope access is required for class decorators to be able to 383 | evaluate type annotations. This is because class decorators are applied 384 | before the class receives its name in the outer scope. 385 | 386 | Introducing a new dictionary for the string literal form instead 387 | ---------------------------------------------------------------- 388 | 389 | Yury Selivanov shared the following idea: 390 | 391 | 1. Add a new special attribute to functions: ``__annotations_text__``. 392 | 393 | 2. Make ``__annotations__`` a lazy dynamic mapping, evaluating 394 | expressions from the corresponding key in ``__annotations_text__`` 395 | just-in-time. 396 | 397 | This idea is supposed to solve the backwards compatibility issue, 398 | removing the need for a new ``__future__`` import. Sadly, this is not 399 | enough. Postponed evaluation changes which state the annotation has 400 | access to. While postponed evaluation fixes the forward reference 401 | problem, it also makes it impossible to access function-level locals 402 | anymore. This alone is a source of backwards incompatibility which 403 | justifies a deprecation period. 404 | 405 | A ``__future__`` import is an obvious and explicit indicator of opting 406 | in for the new functionality. It also makes it trivial for external 407 | tools to recognize the difference between a Python files using the old 408 | or the new approach. In the former case, that tool would recognize that 409 | local state access is allowed, whereas in the latter case it would 410 | recognize that forward references are allowed. 411 | 412 | Finally, just-in-time evaluation in ``__annotations__`` is an 413 | unnecessary step if ``get_type_hints()`` is used later. 414 | 415 | Dropping annotations with -O 416 | ---------------------------- 417 | 418 | There are two reasons this is not satisfying for the purpose of this 419 | PEP. 420 | 421 | First, this only addresses runtime cost, not forward references, those 422 | still cannot be safely used in source code. A library maintainer would 423 | never be able to use forward references since that would force the 424 | library users to use this new hypothetical -O switch. 425 | 426 | Second, this throws the baby out with the bath water. Now *no* runtime 427 | annotation use can be performed. PEP 557 is one example of a recent 428 | development where evaluating type annotations at runtime is useful. 429 | 430 | All that being said, a granular -O option to drop annotations is 431 | a possibility in the future, as it's conceptually compatible with 432 | existing -O behavior (dropping docstrings and assert statements). This 433 | PEP does not invalidate the idea. 434 | 435 | Passing string literals in annotations verbatim to ``__annotations__`` 436 | ---------------------------------------------------------------------- 437 | 438 | This PEP originally suggested directly storing the contents of a string 439 | literal under its respective key in ``__annotations__``. This was 440 | meant to simplify support for runtime type checkers. 441 | 442 | Mark Shannon pointed out this idea was flawed since it wasn't handling 443 | situations where strings are only part of a type annotation. 444 | 445 | The inconsistency of it was always apparent but given that it doesn't 446 | fully prevent cases of double-wrapping strings anyway, it is not worth 447 | it. 448 | 449 | Making the name of the future import more verbose 450 | ------------------------------------------------- 451 | 452 | Instead of requiring the following import:: 453 | 454 | from __future__ import annotations 455 | 456 | the PEP could call the feature more explicitly, for example 457 | ``string_annotations``, ``stringify_annotations``, 458 | ``annotation_strings``, ``annotations_as_strings``, ``lazy_anotations``, 459 | ``static_annotations``, etc. 460 | 461 | The problem with those names is that they are very verbose. Each of 462 | them besides ``lazy_annotations`` would constitute the longest future 463 | feature name in Python. They are long to type and harder to remember 464 | than the single-word form. 465 | 466 | There is precedence of a future import name that sounds overly generic 467 | but in practice was obvious to users as to what it does:: 468 | 469 | from __future__ import division 470 | 471 | 472 | Prior discussion 473 | ================ 474 | 475 | In PEP 484 476 | ---------- 477 | 478 | The forward reference problem was discussed when PEP 484 was originally 479 | drafted, leading to the following statement in the document: 480 | 481 | A compromise is possible where a ``__future__`` import could enable 482 | turning *all* annotations in a given module into string literals, as 483 | follows:: 484 | 485 | from __future__ import annotations 486 | 487 | class ImSet: 488 | def add(self, a: ImSet) -> List[ImSet]: ... 489 | 490 | assert ImSet.add.__annotations__ == { 491 | 'a': 'ImSet', 'return': 'List[ImSet]' 492 | } 493 | 494 | Such a ``__future__`` import statement may be proposed in a separate 495 | PEP. 496 | 497 | python/typing#400 498 | ----------------- 499 | 500 | The problem was discussed at length on the typing module's GitHub 501 | project, under `Issue 400 `_. 502 | The problem statement there includes critique of generic types requiring 503 | imports from ``typing``. This tends to be confusing to 504 | beginners: 505 | 506 | Why this:: 507 | 508 | from typing import List, Set 509 | def dir(o: object = ...) -> List[str]: ... 510 | def add_friends(friends: Set[Friend]) -> None: ... 511 | 512 | But not this:: 513 | 514 | def dir(o: object = ...) -> list[str]: ... 515 | def add_friends(friends: set[Friend]) -> None ... 516 | 517 | Why this:: 518 | 519 | up_to_ten = list(range(10)) 520 | friends = set() 521 | 522 | But not this:: 523 | 524 | from typing import List, Set 525 | up_to_ten = List[int](range(10)) 526 | friends = Set[Friend]() 527 | 528 | While typing usability is an interesting problem, it is out of scope 529 | of this PEP. Specifically, any extensions of the typing syntax 530 | standardized in PEP 484 will require their own respective PEPs and 531 | approval. 532 | 533 | Issue 400 ultimately suggests postponing evaluation of annotations and 534 | keeping them as strings in ``__annotations__``, just like this PEP 535 | specifies. This idea was received well. Ivan Levkivskyi supported 536 | using the ``__future__`` import and suggested unparsing the AST in 537 | ``compile.c``. Jukka Lehtosalo pointed out that there are some cases 538 | of forward references where types are used outside of annotations and 539 | postponed evaluation will not help those. For those cases using the 540 | string literal notation would still be required. Those cases are 541 | discussed briefly in the "Forward References" section of this PEP. 542 | 543 | The biggest controversy on the issue was Guido van Rossum's concern 544 | that untokenizing annotation expressions back to their string form has 545 | no precedent in the Python programming language and feels like a hacky 546 | workaround. He said: 547 | 548 | One thing that comes to mind is that it's a very random change to 549 | the language. It might be useful to have a more compact way to 550 | indicate deferred execution of expressions (using less syntax than 551 | ``lambda:``). But why would the use case of type annotations be so 552 | all-important to change the language to do it there first (rather 553 | than proposing a more general solution), given that there's already 554 | a solution for this particular use case that requires very minimal 555 | syntax? 556 | 557 | Eventually, Ethan Smith and schollii voiced that feedback gathered 558 | during PyCon US suggests that the state of forward references needs 559 | fixing. Guido van Rossum suggested coming back to the ``__future__`` 560 | idea, pointing out that to prevent abuse, it's important for the 561 | annotations to be kept both syntactically valid and evaluating correctly 562 | at runtime. 563 | 564 | First draft discussion on python-ideas 565 | -------------------------------------- 566 | 567 | Discussion happened largely in two threads, `the original announcement 568 | `_ 569 | and a follow-up called `PEP 563 and expensive backwards compatibility 570 | `_. 571 | 572 | The PEP received rather warm feedback (4 strongly in favor, 573 | 2 in favor with concerns, 2 against). The biggest voice of concern on 574 | the former thread being Steven D'Aprano's review stating that the 575 | problem definition of the PEP doesn't justify breaking backwards 576 | compatibility. In this response Steven seemed mostly concerned about 577 | Python no longer supporting evaluation of annotations that depended on 578 | local function/class state. 579 | 580 | A few people voiced concerns that there are libraries using annotations 581 | for non-typing purposes. However, none of the named libraries would be 582 | invalidated by this PEP. They do require adapting to the new 583 | requirement to call ``eval()`` on the annotation with the correct 584 | ``globals`` and ``locals`` set. 585 | 586 | This detail about ``globals`` and ``locals`` having to be correct was 587 | picked up by a number of commenters. Nick Coghlan benchmarked turning 588 | annotations into lambdas instead of strings, sadly this proved to be 589 | much slower at runtime than the current situation. 590 | 591 | The latter thread was started by Jim J. Jewett who stressed that 592 | the ability to properly evaluate annotations is an important requirement 593 | and backwards compatibility in that regard is valuable. After some 594 | discussion he admitted that side effects in annotations are a code smell 595 | and modal support to either perform or not perform evaluation is 596 | a messy solution. His biggest concern remained loss of functionality 597 | stemming from the evaluation restrictions on global and local scope. 598 | 599 | Nick Coghlan pointed out that some of those evaluation restrictions from 600 | the PEP could be lifted by a clever implementation of an evaluation 601 | helper, which could solve self-referencing classes even in the form of a 602 | class decorator. He suggested the PEP should provide this helper 603 | function in the standard library. 604 | 605 | Second draft discussion on python-dev 606 | ------------------------------------- 607 | 608 | Discussion happened mainly in the `announcement thread `_, 609 | followed by a brief discussion under `Mark Shannon's post 610 | `_. 611 | 612 | Steven D'Aprano was concerned whether it's acceptable for typos to be 613 | allowed in annotations after the change proposed by the PEP. Brett 614 | Cannon responded that type checkers and other static analyzers (like 615 | linters or programming text editors) will catch this type of error. 616 | Jukka Lehtosalo added that this situation is analogous to how names in 617 | function bodies are not resolved until the function is called. 618 | 619 | A major topic of discussion was Nick Coghlan's suggestion to store 620 | annotations in "thunk form", in other words as a specialized lambda 621 | which would be able to access class-level scope (and allow for scope 622 | customization at call time). He presented a possible design for it 623 | (`indirect attribute cells 624 | `_). 625 | This was later seen as equivalent to "special forms" in Lisp. Guido van 626 | Rossum expressed worry that this sort of feature cannot be safely 627 | implemented in twelve weeks (i.e. in time before the Python 3.7 beta 628 | freeze). 629 | 630 | After a while it became clear that the point of division between 631 | supporters of the string form vs. supporters of the thunk form is 632 | actually about whether annotations should be perceived as a general 633 | syntactic element vs. something tied to the type checking use case. 634 | 635 | Finally, Guido van Rossum declared he's rejecting the thunk idea 636 | based on the fact that it would require a new building block in the 637 | interpreter. This block would be exposed in annotations, multiplying 638 | possible types of values stored in ``__annotations__`` (arbitrary 639 | objects, strings, and now thunks). Moreover, thunks aren't as 640 | introspectable as strings. Most importantly, Guido van Rossum 641 | explicitly stated interest in gradually restricting the use of 642 | annotations to static typing (with an optional runtime component). 643 | 644 | Nick Coghlan got convinced to PEP 563, too, promptly beginning 645 | the mandatory bike shedding session on the name of the ``__future__`` 646 | import. Many debaters agreed that ``annotations`` seems like 647 | an overly broad name for the feature name. Guido van Rossum briefly 648 | decided to call it ``string_annotations`` but then changed his mind, 649 | arguing that ``division`` is a precedent of a broad name with a clear 650 | meaning. 651 | 652 | The final improvement to the PEP suggested in the discussion by Mark 653 | Shannon was the rejection of the temptation to pass string literals 654 | through to ``__annotations__`` verbatim. 655 | 656 | A side-thread of discussion started around the runtime penalty of 657 | static typing, with topic like the import time of the ``typing`` 658 | module (which is comparable to ``re`` without dependencies, and 659 | three times as heavy as ``re`` when counting dependencies). 660 | 661 | 662 | Acknowledgements 663 | ================ 664 | 665 | This document could not be completed without valuable input, 666 | encouragement and advice from Guido van Rossum, Jukka Lehtosalo, and 667 | Ivan Levkivskyi. 668 | 669 | 670 | Copyright 671 | ========= 672 | 673 | This document has been placed in the public domain. 674 | 675 | 676 | 677 | .. 678 | Local Variables: 679 | mode: indented-text 680 | indent-tabs-mode: nil 681 | sentence-end-double-space: t 682 | fill-column: 70 683 | coding: utf-8 684 | End: 685 | --------------------------------------------------------------------------------