├── .gitignore ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── lazyasd-py2.py ├── lazyasd-py3.py ├── news ├── TEMPLATE.rst └── plus.rst ├── rever.xsh ├── setup.py └── tests └── test_lazyasd.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Custom files 2 | lazyasd.py 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # IPython Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # dotenv 82 | .env 83 | 84 | # virtualenv 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | 91 | # Rope project settings 92 | .ropeproject 93 | 94 | feedstock/ 95 | *.cred 96 | # Rever 97 | rever/ 98 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | lazyasd Change Log 3 | ================== 4 | 5 | .. current developments 6 | 7 | v0.1.4 8 | ==================== 9 | 10 | **Added:** 11 | 12 | * Python 2.7 is now supported! 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, xonsh 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of lazyasd nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include lazyasd-py*.py 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | lazyasd 3 | ======= 4 | A package that provides lazy and self-destructive tools for speeding up module 5 | imports. This is useful whenever startup times are critical, such as for 6 | command line interfaces or other user-facing applications. 7 | 8 | The tools in this module implement two distinct strategies for speeding up 9 | module import. The first is delayed construction of global state and the 10 | second is to import expensive modules in a background thread. 11 | 12 | Feel free to use lazyasd as a dependency or, because it is implemented as a 13 | single module, copy the ``lazyasd.py`` file into your project. 14 | 15 | Lazy Construction 16 | ***************** 17 | Many operations related to data construction or inspection setup can take 18 | a long time to complete. If only a single copy of the data or a cached 19 | representation is needed, in Python it is common to move the data to the 20 | global or module level scope. 21 | 22 | By moving to module level, we help ensure that only a single copy of the data 23 | is ever built. However, by moving to module scope, the single performance hit 24 | now comes at import time. This is itself wasteful if the data is never used. 25 | Furthermore, the more data that is built globally, the longer importing the 26 | module takes. 27 | 28 | For example, consider a function that reports if a string contains the word 29 | ``"foo"`` using regular expressions. The naive version is relatively slow, per 30 | function call, because it has to construct the regex each time: 31 | 32 | .. code-block:: python 33 | 34 | import re 35 | 36 | def has_foo_simple(s): 37 | return re.search('foo', s) is not None 38 | 39 | The standard way of improving performance is to compile the regex at global 40 | scope. Rewriting, we would see: 41 | 42 | .. code-block:: python 43 | 44 | import re 45 | 46 | FOO_RE = re.compile('foo') 47 | 48 | def has_foo_compiled(s): 49 | return FOO_RE.search(s) is not None 50 | 51 | Now, each call of ``has_foo_compiled()`` is much faster than a call of 52 | ``has_foo_simple()`` because we have shifted the compilation to import 53 | time. But what if we never actually call ``has_foo()``? In this case, 54 | the original version was better because the imports are fast. 55 | 56 | Having the best of both compile-once and don't-compile-on-import is where 57 | the lazy and self-destructive tools come in. A ``LazyObject`` instance 58 | has a loader function, a context to place the result of the into, and the 59 | name of the loaded value in the context. The ``LazyObject`` does no 60 | work when it is first created. However, whenever an attribute is accessed 61 | (or a variety of other operations) the loader will be called, the true 62 | value will be constructed, and the ``LazyObject`` will act as a proxy to 63 | loaded object. 64 | 65 | Using the above regex example, we have minimal import-time and run-time 66 | performance hits with the following lazy implementation: 67 | 68 | .. code-block:: python 69 | 70 | import re 71 | from lazyasd import LazyObject 72 | 73 | FOO_RE = LazyObject(lambda: re.compile('foo'), globals(), 'FOO_RE') 74 | 75 | def has_foo_lazy(s): 76 | return FOO_RE.search(s) is not None 77 | 78 | To walk through the above, at import time ``FOO_RE`` is a LazyObject, that has a 79 | lambda loader which returns the regex we care about. If ``FOO_RE`` is never 80 | accessed this is how it will remain. However, the first time ``has_foo_lazy()`` 81 | is called, accessing the ``search`` method will cause the ``LazyObject`` to: 82 | 83 | 1. Call the loader (getting ``re.compile('foo')`` as the result) 84 | 2. Place the result in the context, eg ``globals()['FOO_RE'] = re.compile('foo')`` 85 | 3. Look up attributes and methods (such as ``search``) on the result. 86 | 87 | Now because of the context replacement, ``FOO_RE`` now is a regular expression 88 | object. Further calls to ``has_foo_lazy()`` will see ``FOO_RE`` as a regular 89 | expression object directly, and not as a ``LazyObject``. In fact, if no lingering 90 | references remain, the original ``LazyObject`` instance can be totally cleaned up 91 | by the garbage collector! 92 | 93 | For the truly lazy, there is also a ``lazyobject`` decorator: 94 | 95 | .. code-block:: python 96 | 97 | import re 98 | from lazyasd import lazyobject 99 | 100 | @lazyobject 101 | def foo_re(): 102 | return re.compile('foo') 103 | 104 | def has_foo_lazy(s): 105 | return foo_re.search(s) is not None 106 | 107 | Another useful pattern is to implement lazy module imports, where the 108 | module is only imported if a member of it used: 109 | 110 | .. code-block:: python 111 | 112 | import importlib 113 | from lazyasd import lazyobject 114 | 115 | @lazyobject 116 | def os(): 117 | return importlib.import_module('os') 118 | 119 | The world is beautifully yours, but feel free to take a nap first. 120 | 121 | Specific Laziness 122 | ----------------- 123 | The ``LazyBool`` class and ``lazybool`` decorator have the same interface as 124 | lazy objects. These are provided for objects that are intended to be resolved 125 | as booleans. 126 | 127 | The ``LazyDict`` class and ``lazydict`` decorator are similar. Here however, 128 | the first value is a dictionary of key-loaders. Rather than having a single 129 | loader, each value is loaded individually when its key is first accessed. 130 | 131 | 132 | Background Imports 133 | ****************** 134 | Even with all of the above laziness, sometimes it isn't enough. Sometimes a 135 | module is so painful to import and so unavoidable that you need to import 136 | it on background thread so that the rest of the application can boot up 137 | in the meantime. This is the purpose of ``load_module_in_background()``. 138 | 139 | For example, if you are using pygments and you want the import to safely 140 | be 100x faster, simply drop in the following lines: 141 | 142 | .. code-block:: python 143 | 144 | # must come before pygments imports 145 | from lazyasd import load_module_in_background 146 | load_module_in_background('pkg_resources', 147 | replacements={'pygments.plugin': 'pkg_resources'}) 148 | 149 | # now pygments is fast to import 150 | from pygments.style import Style 151 | 152 | This prevents ``pkg_resources``, which comes from setuptools, from searching your 153 | entire filesystem for plugins at import time. Like above, this import acts as 154 | proxy and will block until it is needed. It is also robust if the module has 155 | already been imported. In some cases, this background importing is the best a 156 | third party application can do. 157 | 158 | -------------------------------------------------------------------------------- /lazyasd-py2.py: -------------------------------------------------------------------------------- 1 | """Lazy and self destructive containers for speeding up module import.""" 2 | # Copyright 2015-2016, the xonsh developers. All rights reserved. 3 | import os 4 | import sys 5 | import time 6 | import types 7 | import threading 8 | import importlib 9 | try: 10 | from importlib.util import resolve_name 11 | except ImportError: 12 | def resolve_name(name, package): 13 | if name.startswith('.'): 14 | raise NotImplementedError('Python 3 needed for relative modules') 15 | return name 16 | try: 17 | from collections.abc import MutableMapping 18 | except ImportError: 19 | from collections import MutableMapping 20 | 21 | __version__ = '0.1.4' 22 | 23 | 24 | class LazyObject(object): 25 | 26 | def __init__(self, load, ctx, name): 27 | """Lazily loads an object via the load function the first time an 28 | attribute is accessed. Once loaded it will replace itself in the 29 | provided context (typically the globals of the call site) with the 30 | given name. 31 | 32 | For example, you can prevent the compilation of a regular expreession 33 | until it is actually used:: 34 | 35 | DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT') 36 | 37 | Parameters 38 | ---------- 39 | load : function with no arguments 40 | A loader function that performs the actual object construction. 41 | ctx : Mapping 42 | Context to replace the LazyObject instance in 43 | with the object returned by load(). 44 | name : str 45 | Name in the context to give the loaded object. This *should* 46 | be the name on the LHS of the assignment. 47 | """ 48 | self._lasdo = { 49 | 'loaded': False, 50 | 'load': load, 51 | 'ctx': ctx, 52 | 'name': name, 53 | } 54 | 55 | def _lazy_obj(self): 56 | d = self._lasdo 57 | if d['loaded']: 58 | obj = d['obj'] 59 | else: 60 | obj = d['load']() 61 | d['ctx'][d['name']] = d['obj'] = obj 62 | d['loaded'] = True 63 | return obj 64 | 65 | def __getattribute__(self, name): 66 | if name == '_lasdo' or name == '_lazy_obj': 67 | return super(LazyObject, self).__getattribute__(name) 68 | obj = self._lazy_obj() 69 | return getattr(obj, name) 70 | 71 | def __add__(self, other): 72 | obj = self._lazy_obj() 73 | return obj + other 74 | 75 | def __radd__(self, other): 76 | obj = self._lazy_obj() 77 | return other + obj 78 | 79 | def __bool__(self): 80 | obj = self._lazy_obj() 81 | return bool(obj) 82 | 83 | def __iter__(self): 84 | obj = self._lazy_obj() 85 | for item in obj: 86 | yield item 87 | 88 | def __getitem__(self, item): 89 | obj = self._lazy_obj() 90 | return obj[item] 91 | 92 | def __setitem__(self, key, value): 93 | obj = self._lazy_obj() 94 | obj[key] = value 95 | 96 | def __delitem__(self, item): 97 | obj = self._lazy_obj() 98 | del obj[item] 99 | 100 | def __call__(self, *args, **kwargs): 101 | obj = self._lazy_obj() 102 | return obj(*args, **kwargs) 103 | 104 | def __lt__(self, other): 105 | obj = self._lazy_obj() 106 | return obj < other 107 | 108 | def __le__(self, other): 109 | obj = self._lazy_obj() 110 | return obj <= other 111 | 112 | def __eq__(self, other): 113 | obj = self._lazy_obj() 114 | return obj == other 115 | 116 | def __ne__(self, other): 117 | obj = self._lazy_obj() 118 | return obj != other 119 | 120 | def __gt__(self, other): 121 | obj = self._lazy_obj() 122 | return obj > other 123 | 124 | def __ge__(self, other): 125 | obj = self._lazy_obj() 126 | return obj >= other 127 | 128 | def __hash__(self): 129 | obj = self._lazy_obj() 130 | return hash(obj) 131 | 132 | def __or__(self, other): 133 | obj = self._lazy_obj() 134 | return obj | other 135 | 136 | def __str__(self): 137 | return str(self._lazy_obj()) 138 | 139 | def __repr__(self): 140 | return repr(self._lazy_obj()) 141 | 142 | 143 | def lazyobject(f): 144 | """Decorator for constructing lazy objects from a function.""" 145 | return LazyObject(f, f.__globals__, f.__name__) 146 | 147 | 148 | class LazyDict(MutableMapping): 149 | 150 | def __init__(self, loaders, ctx, name): 151 | """Dictionary like object that lazily loads its values from an initial 152 | dict of key-loader function pairs. Each key is loaded when its value 153 | is first accessed. Once fully loaded, this object will replace itself 154 | in the provided context (typically the globals of the call site) with 155 | the given name. 156 | 157 | For example, you can prevent the compilation of a bunch of regular 158 | expressions until they are actually used:: 159 | 160 | RES = LazyDict({ 161 | 'dot': lambda: re.compile('.'), 162 | 'all': lambda: re.compile('.*'), 163 | 'two': lambda: re.compile('..'), 164 | }, globals(), 'RES') 165 | 166 | Parameters 167 | ---------- 168 | loaders : Mapping of keys to functions with no arguments 169 | A mapping of loader function that performs the actual value 170 | construction upon acces. 171 | ctx : Mapping 172 | Context to replace the LazyDict instance in 173 | with the the fully loaded mapping. 174 | name : str 175 | Name in the context to give the loaded mapping. This *should* 176 | be the name on the LHS of the assignment. 177 | """ 178 | self._loaders = loaders 179 | self._ctx = ctx 180 | self._name = name 181 | self._d = type(loaders)() # make sure to return the same type 182 | 183 | def _destruct(self): 184 | if len(self._loaders) == 0: 185 | self._ctx[self._name] = self._d 186 | 187 | def __getitem__(self, key): 188 | d = self._d 189 | if key in d: 190 | val = d[key] 191 | else: 192 | # pop will raise a key error for us 193 | loader = self._loaders.pop(key) 194 | d[key] = val = loader() 195 | self._destruct() 196 | return val 197 | 198 | def __setitem__(self, key, value): 199 | self._d[key] = value 200 | if key in self._loaders: 201 | del self._loaders[key] 202 | self._destruct() 203 | 204 | def __delitem__(self, key): 205 | if key in self._d: 206 | del self._d[key] 207 | else: 208 | del self._loaders[key] 209 | self._destruct() 210 | 211 | def __iter__(self): 212 | for item in (set(self._d.keys()) | set(self._loaders.keys())): 213 | yield item 214 | 215 | def __len__(self): 216 | return len(self._d) + len(self._loaders) 217 | 218 | 219 | def lazydict(f): 220 | """Decorator for constructing lazy dicts from a function.""" 221 | return LazyDict(f, f.__globals__, f.__name__) 222 | 223 | 224 | class LazyBool(object): 225 | 226 | def __init__(self, load, ctx, name): 227 | """Boolean like object that lazily computes it boolean value when it is 228 | first asked. Once loaded, this result will replace itself 229 | in the provided context (typically the globals of the call site) with 230 | the given name. 231 | 232 | For example, you can prevent the complex boolean until it is actually 233 | used:: 234 | 235 | ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE') 236 | 237 | Parameters 238 | ---------- 239 | load : function with no arguments 240 | A loader function that performs the actual boolean evaluation. 241 | ctx : Mapping 242 | Context to replace the LazyBool instance in 243 | with the the fully loaded mapping. 244 | name : str 245 | Name in the context to give the loaded mapping. This *should* 246 | be the name on the LHS of the assignment. 247 | """ 248 | self._load = load 249 | self._ctx = ctx 250 | self._name = name 251 | self._result = None 252 | 253 | def __bool__(self): 254 | if self._result is None: 255 | res = self._ctx[self._name] = self._result = self._load() 256 | else: 257 | res = self._result 258 | return res 259 | 260 | 261 | def lazybool(f): 262 | """Decorator for constructing lazy booleans from a function.""" 263 | return LazyBool(f, f.__globals__, f.__name__) 264 | 265 | 266 | # 267 | # Background module loaders 268 | # 269 | 270 | class BackgroundModuleProxy(types.ModuleType): 271 | """Proxy object for modules loaded in the background that block attribute 272 | access until the module is loaded.. 273 | """ 274 | 275 | def __init__(self, modname): 276 | self.__dct__ = { 277 | 'loaded': False, 278 | 'modname': modname, 279 | } 280 | 281 | def __getattribute__(self, name): 282 | passthrough = frozenset({'__dct__', '__class__', '__spec__'}) 283 | if name in passthrough: 284 | return super(BackgroundModuleProxy, self).__getattribute__(name) 285 | dct = self.__dct__ 286 | modname = dct['modname'] 287 | if dct['loaded']: 288 | mod = sys.modules[modname] 289 | else: 290 | delay_types = (BackgroundModuleProxy, type(None)) 291 | while isinstance(sys.modules.get(modname, None), delay_types): 292 | time.sleep(0.001) 293 | mod = sys.modules[modname] 294 | dct['loaded'] = True 295 | # some modules may do construction after import, give them a second 296 | stall = 0 297 | while not hasattr(mod, name) and stall < 1000: 298 | stall += 1 299 | time.sleep(0.001) 300 | return getattr(mod, name) 301 | 302 | 303 | class BackgroundModuleLoader(threading.Thread): 304 | """Thread to load modules in the background.""" 305 | 306 | def __init__(self, name, package, replacements, *args, **kwargs): 307 | super(BackgroundModuleLoader, self).__init__(*args, **kwargs) 308 | self.daemon = True 309 | self.name = name 310 | self.package = package 311 | self.replacements = replacements 312 | self.start() 313 | 314 | def run(self): 315 | # wait for other modules to stop being imported 316 | # We assume that module loading is finished when sys.modules doesn't 317 | # get longer in 5 consecutive 1ms waiting steps 318 | counter = 0 319 | last = -1 320 | while counter < 5: 321 | new = len(sys.modules) 322 | if new == last: 323 | counter += 1 324 | else: 325 | last = new 326 | counter = 0 327 | time.sleep(0.001) 328 | # now import module properly 329 | modname = resolve_name(self.name, self.package) 330 | if isinstance(sys.modules[modname], BackgroundModuleProxy): 331 | del sys.modules[modname] 332 | mod = importlib.import_module(self.name, package=self.package) 333 | for targname, varname in self.replacements.items(): 334 | if targname in sys.modules: 335 | targmod = sys.modules[targname] 336 | setattr(targmod, varname, mod) 337 | 338 | 339 | def load_module_in_background(name, package=None, debug='DEBUG', env=None, 340 | replacements=None): 341 | """Entry point for loading modules in background thread. 342 | 343 | Parameters 344 | ---------- 345 | name : str 346 | Module name to load in background thread. 347 | package : str or None, optional 348 | Package name, has the same meaning as in importlib.import_module(). 349 | debug : str, optional 350 | Debugging symbol name to look up in the environment. 351 | env : Mapping or None, optional 352 | Environment this will default to __xonsh_env__, if available, and 353 | os.environ otherwise. 354 | replacements : Mapping or None, optional 355 | Dictionary mapping fully qualified module names (eg foo.bar.baz) that 356 | import the lazily loaded moudle, with the variable name in that 357 | module. For example, suppose that foo.bar imports module a as b, 358 | this dict is then {'foo.bar': 'b'}. 359 | 360 | Returns 361 | ------- 362 | module : ModuleType 363 | This is either the original module that is found in sys.modules or 364 | a proxy module that will block until delay attribute access until the 365 | module is fully loaded. 366 | """ 367 | modname = resolve_name(name, package) 368 | if modname in sys.modules: 369 | return sys.modules[modname] 370 | if env is None: 371 | try: 372 | import builtins 373 | env = getattr(builtins, '__xonsh_env__', os.environ) 374 | except: 375 | return os.environ 376 | if env.get(debug, None): 377 | mod = importlib.import_module(name, package=package) 378 | return mod 379 | proxy = sys.modules[modname] = BackgroundModuleProxy(modname) 380 | BackgroundModuleLoader(name, package, replacements or {}) 381 | return proxy 382 | -------------------------------------------------------------------------------- /lazyasd-py3.py: -------------------------------------------------------------------------------- 1 | """Lazy and self destructive containers for speeding up module import.""" 2 | # Copyright 2015-2016, the xonsh developers. All rights reserved. 3 | import os 4 | import sys 5 | import time 6 | import types 7 | import builtins 8 | import threading 9 | import importlib 10 | import importlib.util 11 | import collections.abc as cabc 12 | 13 | __version__ = '0.1.4' 14 | 15 | 16 | class LazyObject(object): 17 | 18 | def __init__(self, load, ctx, name): 19 | """Lazily loads an object via the load function the first time an 20 | attribute is accessed. Once loaded it will replace itself in the 21 | provided context (typically the globals of the call site) with the 22 | given name. 23 | 24 | For example, you can prevent the compilation of a regular expreession 25 | until it is actually used:: 26 | 27 | DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT') 28 | 29 | Parameters 30 | ---------- 31 | load : function with no arguments 32 | A loader function that performs the actual object construction. 33 | ctx : Mapping 34 | Context to replace the LazyObject instance in 35 | with the object returned by load(). 36 | name : str 37 | Name in the context to give the loaded object. This *should* 38 | be the name on the LHS of the assignment. 39 | """ 40 | self._lasdo = { 41 | 'loaded': False, 42 | 'load': load, 43 | 'ctx': ctx, 44 | 'name': name, 45 | } 46 | 47 | def _lazy_obj(self): 48 | d = self._lasdo 49 | if d['loaded']: 50 | obj = d['obj'] 51 | else: 52 | obj = d['load']() 53 | d['ctx'][d['name']] = d['obj'] = obj 54 | d['loaded'] = True 55 | return obj 56 | 57 | def __getattribute__(self, name): 58 | if name == '_lasdo' or name == '_lazy_obj': 59 | return super().__getattribute__(name) 60 | obj = self._lazy_obj() 61 | return getattr(obj, name) 62 | 63 | def __add__(self, other): 64 | obj = self._lazy_obj() 65 | return obj + other 66 | 67 | def __radd__(self, other): 68 | obj = self._lazy_obj() 69 | return other + obj 70 | 71 | def __bool__(self): 72 | obj = self._lazy_obj() 73 | return bool(obj) 74 | 75 | def __iter__(self): 76 | obj = self._lazy_obj() 77 | yield from obj 78 | 79 | def __getitem__(self, item): 80 | obj = self._lazy_obj() 81 | return obj[item] 82 | 83 | def __setitem__(self, key, value): 84 | obj = self._lazy_obj() 85 | obj[key] = value 86 | 87 | def __delitem__(self, item): 88 | obj = self._lazy_obj() 89 | del obj[item] 90 | 91 | def __call__(self, *args, **kwargs): 92 | obj = self._lazy_obj() 93 | return obj(*args, **kwargs) 94 | 95 | def __lt__(self, other): 96 | obj = self._lazy_obj() 97 | return obj < other 98 | 99 | def __le__(self, other): 100 | obj = self._lazy_obj() 101 | return obj <= other 102 | 103 | def __eq__(self, other): 104 | obj = self._lazy_obj() 105 | return obj == other 106 | 107 | def __ne__(self, other): 108 | obj = self._lazy_obj() 109 | return obj != other 110 | 111 | def __gt__(self, other): 112 | obj = self._lazy_obj() 113 | return obj > other 114 | 115 | def __ge__(self, other): 116 | obj = self._lazy_obj() 117 | return obj >= other 118 | 119 | def __hash__(self): 120 | obj = self._lazy_obj() 121 | return hash(obj) 122 | 123 | def __or__(self, other): 124 | obj = self._lazy_obj() 125 | return obj | other 126 | 127 | def __str__(self): 128 | return str(self._lazy_obj()) 129 | 130 | def __repr__(self): 131 | return repr(self._lazy_obj()) 132 | 133 | 134 | def lazyobject(f): 135 | """Decorator for constructing lazy objects from a function.""" 136 | return LazyObject(f, f.__globals__, f.__name__) 137 | 138 | 139 | class LazyDict(cabc.MutableMapping): 140 | 141 | def __init__(self, loaders, ctx, name): 142 | """Dictionary like object that lazily loads its values from an initial 143 | dict of key-loader function pairs. Each key is loaded when its value 144 | is first accessed. Once fully loaded, this object will replace itself 145 | in the provided context (typically the globals of the call site) with 146 | the given name. 147 | 148 | For example, you can prevent the compilation of a bunch of regular 149 | expressions until they are actually used:: 150 | 151 | RES = LazyDict({ 152 | 'dot': lambda: re.compile('.'), 153 | 'all': lambda: re.compile('.*'), 154 | 'two': lambda: re.compile('..'), 155 | }, globals(), 'RES') 156 | 157 | Parameters 158 | ---------- 159 | loaders : Mapping of keys to functions with no arguments 160 | A mapping of loader function that performs the actual value 161 | construction upon acces. 162 | ctx : Mapping 163 | Context to replace the LazyDict instance in 164 | with the the fully loaded mapping. 165 | name : str 166 | Name in the context to give the loaded mapping. This *should* 167 | be the name on the LHS of the assignment. 168 | """ 169 | self._loaders = loaders 170 | self._ctx = ctx 171 | self._name = name 172 | self._d = type(loaders)() # make sure to return the same type 173 | 174 | def _destruct(self): 175 | if len(self._loaders) == 0: 176 | self._ctx[self._name] = self._d 177 | 178 | def __getitem__(self, key): 179 | d = self._d 180 | if key in d: 181 | val = d[key] 182 | else: 183 | # pop will raise a key error for us 184 | loader = self._loaders.pop(key) 185 | d[key] = val = loader() 186 | self._destruct() 187 | return val 188 | 189 | def __setitem__(self, key, value): 190 | self._d[key] = value 191 | if key in self._loaders: 192 | del self._loaders[key] 193 | self._destruct() 194 | 195 | def __delitem__(self, key): 196 | if key in self._d: 197 | del self._d[key] 198 | else: 199 | del self._loaders[key] 200 | self._destruct() 201 | 202 | def __iter__(self): 203 | yield from (set(self._d.keys()) | set(self._loaders.keys())) 204 | 205 | def __len__(self): 206 | return len(self._d) + len(self._loaders) 207 | 208 | 209 | def lazydict(f): 210 | """Decorator for constructing lazy dicts from a function.""" 211 | return LazyDict(f, f.__globals__, f.__name__) 212 | 213 | 214 | class LazyBool(object): 215 | 216 | def __init__(self, load, ctx, name): 217 | """Boolean like object that lazily computes it boolean value when it is 218 | first asked. Once loaded, this result will replace itself 219 | in the provided context (typically the globals of the call site) with 220 | the given name. 221 | 222 | For example, you can prevent the complex boolean until it is actually 223 | used:: 224 | 225 | ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE') 226 | 227 | Parameters 228 | ---------- 229 | load : function with no arguments 230 | A loader function that performs the actual boolean evaluation. 231 | ctx : Mapping 232 | Context to replace the LazyBool instance in 233 | with the the fully loaded mapping. 234 | name : str 235 | Name in the context to give the loaded mapping. This *should* 236 | be the name on the LHS of the assignment. 237 | """ 238 | self._load = load 239 | self._ctx = ctx 240 | self._name = name 241 | self._result = None 242 | 243 | def __bool__(self): 244 | if self._result is None: 245 | res = self._ctx[self._name] = self._result = self._load() 246 | else: 247 | res = self._result 248 | return res 249 | 250 | 251 | def lazybool(f): 252 | """Decorator for constructing lazy booleans from a function.""" 253 | return LazyBool(f, f.__globals__, f.__name__) 254 | 255 | 256 | # 257 | # Background module loaders 258 | # 259 | 260 | class BackgroundModuleProxy(types.ModuleType): 261 | """Proxy object for modules loaded in the background that block attribute 262 | access until the module is loaded.. 263 | """ 264 | 265 | def __init__(self, modname): 266 | self.__dct__ = { 267 | 'loaded': False, 268 | 'modname': modname, 269 | } 270 | 271 | def __getattribute__(self, name): 272 | passthrough = frozenset({'__dct__', '__class__', '__spec__'}) 273 | if name in passthrough: 274 | return super().__getattribute__(name) 275 | dct = self.__dct__ 276 | modname = dct['modname'] 277 | if dct['loaded']: 278 | mod = sys.modules[modname] 279 | else: 280 | delay_types = (BackgroundModuleProxy, type(None)) 281 | while isinstance(sys.modules.get(modname, None), delay_types): 282 | time.sleep(0.001) 283 | mod = sys.modules[modname] 284 | dct['loaded'] = True 285 | # some modules may do construction after import, give them a second 286 | stall = 0 287 | while not hasattr(mod, name) and stall < 1000: 288 | stall += 1 289 | time.sleep(0.001) 290 | return getattr(mod, name) 291 | 292 | 293 | class BackgroundModuleLoader(threading.Thread): 294 | """Thread to load modules in the background.""" 295 | 296 | def __init__(self, name, package, replacements, *args, **kwargs): 297 | super().__init__(*args, **kwargs) 298 | self.daemon = True 299 | self.name = name 300 | self.package = package 301 | self.replacements = replacements 302 | self.start() 303 | 304 | def run(self): 305 | # wait for other modules to stop being imported 306 | # We assume that module loading is finished when sys.modules doesn't 307 | # get longer in 5 consecutive 1ms waiting steps 308 | counter = 0 309 | last = -1 310 | while counter < 5: 311 | new = len(sys.modules) 312 | if new == last: 313 | counter += 1 314 | else: 315 | last = new 316 | counter = 0 317 | time.sleep(0.001) 318 | # now import module properly 319 | modname = importlib.util.resolve_name(self.name, self.package) 320 | if isinstance(sys.modules[modname], BackgroundModuleProxy): 321 | del sys.modules[modname] 322 | mod = importlib.import_module(self.name, package=self.package) 323 | for targname, varname in self.replacements.items(): 324 | if targname in sys.modules: 325 | targmod = sys.modules[targname] 326 | setattr(targmod, varname, mod) 327 | 328 | 329 | def load_module_in_background(name, package=None, debug='DEBUG', env=None, 330 | replacements=None): 331 | """Entry point for loading modules in background thread. 332 | 333 | Parameters 334 | ---------- 335 | name : str 336 | Module name to load in background thread. 337 | package : str or None, optional 338 | Package name, has the same meaning as in importlib.import_module(). 339 | debug : str, optional 340 | Debugging symbol name to look up in the environment. 341 | env : Mapping or None, optional 342 | Environment this will default to __xonsh_env__, if available, and 343 | os.environ otherwise. 344 | replacements : Mapping or None, optional 345 | Dictionary mapping fully qualified module names (eg foo.bar.baz) that 346 | import the lazily loaded moudle, with the variable name in that 347 | module. For example, suppose that foo.bar imports module a as b, 348 | this dict is then {'foo.bar': 'b'}. 349 | 350 | Returns 351 | ------- 352 | module : ModuleType 353 | This is either the original module that is found in sys.modules or 354 | a proxy module that will block until delay attribute access until the 355 | module is fully loaded. 356 | """ 357 | modname = importlib.util.resolve_name(name, package) 358 | if modname in sys.modules: 359 | return sys.modules[modname] 360 | if env is None: 361 | env = getattr(builtins, '__xonsh_env__', os.environ) 362 | if env.get(debug, None): 363 | mod = importlib.import_module(name, package=package) 364 | return mod 365 | proxy = sys.modules[modname] = BackgroundModuleProxy(modname) 366 | BackgroundModuleLoader(name, package, replacements or {}) 367 | return proxy 368 | -------------------------------------------------------------------------------- /news/TEMPLATE.rst: -------------------------------------------------------------------------------- 1 | **Added:** None 2 | 3 | **Changed:** None 4 | 5 | **Deprecated:** None 6 | 7 | **Removed:** None 8 | 9 | **Fixed:** None 10 | 11 | **Security:** None 12 | -------------------------------------------------------------------------------- /news/plus.rst: -------------------------------------------------------------------------------- 1 | **Added:** 2 | 3 | * support for plus operator (`+`) via `__add__` and `__radd__` magic methods 4 | 5 | **Changed:** None 6 | 7 | **Deprecated:** None 8 | 9 | **Removed:** None 10 | 11 | **Fixed:** None 12 | 13 | **Security:** None 14 | -------------------------------------------------------------------------------- /rever.xsh: -------------------------------------------------------------------------------- 1 | $PROJECT = 'lazyasd' 2 | $ACTIVITIES = ['version_bump', 'changelog', 'tag', 'push_tag', 'pypi', 'conda_forge', 'ghrelease'] 3 | 4 | $VERSION_BUMP_PATTERNS = [ 5 | ('lazyasd-py2.py', '__version__\s*=.*', "__version__ = '$VERSION'"), 6 | ('lazyasd-py3.py', '__version__\s*=.*', "__version__ = '$VERSION'"), 7 | ('setup.py', 'VERSION\s*=.*', "VERSION = '$VERSION'") 8 | ] 9 | $CHANGELOG_FILENAME = 'CHANGELOG.rst' 10 | $CHANGELOG_IGNORE = ['TEMPLATE.rst'] 11 | $PUSH_TAG_REMOTE = 'git@github.com:xonsh/lazyasd.git' 12 | 13 | $GITHUB_ORG = 'xonsh' 14 | $GITHUB_REPO = 'lazyasd' 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import sys 3 | try: 4 | from setuptools import setup 5 | HAVE_SETUPTOOLS = True 6 | except ImportError: 7 | from distutils.core import setup 8 | HAVE_SETUPTOOLS = False 9 | 10 | 11 | VERSION = '0.1.4' 12 | 13 | # copy over the correct version of lazyasd 14 | filename = 'lazyasd-py3.py' if sys.version_info.major >= 3 else 'lazyasd-py2.py' 15 | lazyasd = """######################################################################## 16 | ### WARNING! Copied from {0} 17 | ### Do not modify directly, instead edit the original file! 18 | ######################################################################## 19 | """.format(filename) 20 | with open(filename) as f: 21 | lazyasd += f.read() 22 | with open('lazyasd.py', 'w') as f: 23 | f.write(lazyasd) 24 | 25 | setup_kwargs = { 26 | "version": VERSION, 27 | "description": ('Lazy & self-destructive tools for speeding up ' 28 | 'module imports'), 29 | "license": 'BSD 3-clause', 30 | "author": 'The xonsh developers', 31 | "author_email": 'xonsh@googlegroups.com', 32 | "url": 'https://github.com/xonsh/lazyasd', 33 | "download_url": "https://github.com/xonsh/lazyasd/zipball/" + VERSION, 34 | "classifiers": [ 35 | "License :: OSI Approved", 36 | "Intended Audience :: Developers", 37 | "Programming Language :: Python", 38 | "Topic :: Utilities", 39 | ], 40 | "zip_safe": False, 41 | } 42 | 43 | 44 | if __name__ == '__main__': 45 | setup( 46 | name='lazyasd', 47 | py_modules=['lazyasd'], 48 | long_description=open('README.rst').read(), 49 | **setup_kwargs 50 | ) 51 | -------------------------------------------------------------------------------- /tests/test_lazyasd.py: -------------------------------------------------------------------------------- 1 | """Tests lazy and self destruictive objects.""" 2 | from lazyasd import LazyObject, load_module_in_background 3 | 4 | # 5 | # LazyObject Tests 6 | # 7 | 8 | def test_lazyobject_getitem(): 9 | lo = LazyObject(lambda: {'x': 1}, {}, 'lo') 10 | assert 1 == lo['x'] 11 | 12 | 13 | def test_bg_load(): 14 | load_module_in_background('pkg_resources') 15 | import pkg_resources 16 | pkg_resources.iter_entry_points 17 | 18 | def test_lazyobject_plus(): 19 | a, s = "a ", "test" 20 | lo = LazyObject(lambda: a, {}, 'lo') 21 | assert lo + s == a + s 22 | assert s + lo == s + a 23 | --------------------------------------------------------------------------------