├── .gitignore ├── LICENSE.txt ├── LICENSES └── ipython_memory_usage.txt ├── README.rst ├── ipython_memwatcher ├── __init__.py └── memwatcher.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Francesc Alted 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /LICENSES/ipython_memory_usage.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Ian Ozsvald 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ipython_memwatcher 2 | ================== 3 | 4 | IPython tool to report memory usage deltas for every command you 5 | type. If you are running out of RAM then use this tool to understand 6 | what's happening. It also records the time spent running each command. 7 | 8 | This tool helps you to figure out which commands use a lot of RAM and 9 | take a long time to run, this is very useful if you're working with 10 | large numpy matrices. In addition it reports the peak memory usage 11 | whilst a command is running which might be higher (due to temporary 12 | objects) than the final RAM usage. Built on @fabianp's 13 | `memory_profiler`. 14 | 15 | As a simple example - make 10,000,000 random numbers, report that it 16 | costs 76MB of RAM and took 0.3 seconds to execute:: 17 | 18 | In [1]: import numpy as np 19 | 20 | In [2]: from ipython_memwatcher import MemWatcher 21 | 22 | In [3]: mw = MemWatcher() 23 | 24 | In [4]: mw.start_watching_memory() 25 | In [4] used 0.0156 MiB RAM in 2.77s, peaked 0.00 MiB above current, total RAM usage 36.27 MiB 26 | 27 | In [5]: arr=np.random.uniform(size=1e7) 28 | In [5] used 76.3320 MiB RAM in 0.33s, peaked 0.00 MiB above current, total RAM usage 112.60 MiB 29 | 30 | And if we also want to have access to the measurements, just call the 31 | `measurements` property:: 32 | 33 | In [6]: mw.measurements 34 | Out[6]: Measurements(memory_delta=76.33203125, time_delta=0.32660794258117676, memory_peak=0, memory_usage=112.59765625) 35 | In [6] used 0.0664 MiB RAM in 0.10s, peaked 0.00 MiB above current, total RAM usage 112.66 MiB 36 | 37 | Works with Python 2.7 and 3.4 or higher, and with IPython 3.0 and up. 38 | 39 | **Note**: This work is strongly based on 40 | https://github.com/ianozsvald/ipython_memory_usage by Ian Ozsvald and 41 | adds basically a handier object interface and a `.measurements` property 42 | for getting access to the actualy memory values. In the future 43 | `ipython_memwatcher` can merged back into `ipython_memory_usage`. 44 | 45 | Example usage 46 | ============= 47 | 48 | We can measure on every line how large array operations allocate and 49 | deallocate memory:: 50 | 51 | In [1]: import numpy as np 52 | 53 | In [2]: from ipython_memwatcher import MemWatcher 54 | 55 | In [3]: mw = MemWatcher() 56 | 57 | In [4]: mw.start_watching_memory() 58 | In [4] used 0.0156 MiB RAM in 5.24s, peaked 0.00 MiB above current, total RAM usage 36.20 MiB 59 | 60 | In [5]: a = np.ones(1e7) 61 | In [5] used 76.3320 MiB RAM in 0.13s, peaked 0.00 MiB above current, total RAM usage 112.53 MiB 62 | 63 | In [6]: b = np.ones(1e7) 64 | In [6] used 76.3203 MiB RAM in 0.12s, peaked 0.00 MiB above current, total RAM usage 188.85 MiB 65 | 66 | In [7]: b = a * b 67 | In [7] used 0.0859 MiB RAM in 0.14s, peaked 2.23 MiB above current, total RAM usage 188.93 MiB 68 | 69 | In [8]: mw.measurements 70 | Out[8]: Measurements(memory_delta=0.0859375, time_delta=0.1445159912109375, memory_peak=2.234375, memory_usage=188.93359375) 71 | In [8] used 0.0703 MiB RAM in 0.10s, peaked 0.00 MiB above current, total RAM usage 189.00 MiB 72 | 73 | You can use `stop_watching_memory` to stop watching and printing 74 | memory usage after each statement:: 75 | 76 | In [9]: mw.stop_watching_memory() 77 | In [10]: b = a * b 78 | 79 | In [11]: 80 | 81 | Important RAM usage note 82 | ======================== 83 | 84 | It is much easier to debug RAM situations with a fresh IPython 85 | shell. The longer you use your current shell, the more objects remain 86 | inside it and the more RAM the Operating System may have reserved. RAM 87 | is returned to the OS slowly, so you can end up with a large process 88 | with plenty of spare internal RAM (which will be allocated to your 89 | large objects), so this tool (via memory_profiler) reports 0MB RAM 90 | usage. If you get confused or don't trust the results, quit IPython 91 | and start a fresh shell, then run the fewest commands you need to 92 | understand how RAM is added to the process. 93 | 94 | Requirements 95 | ============ 96 | 97 | * `memory_profiler` https://github.com/fabianp/memory_profiler 98 | 99 | Tested on 100 | ========= 101 | 102 | * IPython 3.2 with Python 2.7 on Linux 64bit (2015-07) 103 | * IPython 7.2 with Python 3.7 on Mac OSX 64bit (2019-09) 104 | -------------------------------------------------------------------------------- /ipython_memwatcher/__init__.py: -------------------------------------------------------------------------------- 1 | from ipython_memwatcher.memwatcher import MemWatcher 2 | -------------------------------------------------------------------------------- /ipython_memwatcher/memwatcher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Profile mem usage envelope of IPython commands and report interactively""" 4 | from __future__ import division # 1/2 == 0.5, as in Py3 5 | from __future__ import absolute_import # avoid hiding global modules with locals 6 | from __future__ import print_function # force use of print("hello") 7 | from __future__ import unicode_literals # force unadorned strings "" to be unicode without prepending u"" 8 | import time 9 | import memory_profiler 10 | from collections import namedtuple 11 | import threading 12 | from IPython import get_ipython 13 | 14 | __version__ = "0.2.5" 15 | 16 | 17 | class MemWatcher(object): 18 | 19 | def __init__(self): 20 | # keep a global accounting for the last known memory usage 21 | # which is the reference point for the memory delta calculation 22 | self.previous_call_memory_usage = memory_profiler.memory_usage()[0] 23 | self.t1 = time.time() # will be set to current time later 24 | self.keep_watching = True 25 | self.peak_memory_usage = -1 26 | self.peaked_memory_usage = -1 27 | self.memory_delta = 0 28 | self.time_delta = 0 29 | self.watching_memory = True 30 | self.ip = get_ipython() 31 | self.input_cells = self.ip.user_ns['In'] 32 | self._measurements = namedtuple( 33 | 'Measurements', 34 | ['memory_delta', 'time_delta', 'memory_peak', 'memory_usage'], 35 | ) 36 | 37 | @property 38 | def measurements(self): 39 | return self._measurements( 40 | self.memory_delta, self.time_delta, 41 | self.peaked_memory_usage, self.previous_call_memory_usage) 42 | 43 | def start_watching_memory(self): 44 | """Register memory profiling tools to IPython instance.""" 45 | 46 | # Just in case start is called more than once, stop watching. Hence unregister events. 47 | self.stop_watching_memory() 48 | 49 | self.watching_memory = True 50 | self.ip.events.register("post_run_cell", self.watch_memory) 51 | self.ip.events.register("pre_run_cell", self.pre_run_cell) 52 | 53 | def stop_watching_memory(self): 54 | """Unregister memory profiling tools from IPython instance.""" 55 | self.watching_memory = False 56 | try: 57 | self.ip.events.unregister("post_run_cell", self.watch_memory) 58 | except ValueError: 59 | pass 60 | try: 61 | self.ip.events.unregister("pre_run_cell", self.pre_run_cell) 62 | except ValueError: 63 | pass 64 | 65 | def watch_memory(self): 66 | if not self.watching_memory: 67 | return 68 | # calculate time delta using global t1 (from the pre-run 69 | # event) and current time 70 | self.time_delta = time.time() - self.t1 71 | new_memory_usage = memory_profiler.memory_usage()[0] 72 | self.memory_delta = new_memory_usage - self.previous_call_memory_usage 73 | self.keep_watching = False 74 | self.peaked_memory_usage = max(0, self.peak_memory_usage - new_memory_usage) 75 | num_commands = len(self.input_cells) - 1 76 | cmd = "In [{}]".format(num_commands) 77 | # convert the results into a pretty string 78 | output_template = ("{cmd} used {memory_delta:0.3f} MiB RAM in " 79 | "{time_delta:0.3f}s, peaked {peaked_memory_usage:0.3f} " 80 | "MiB above current, total RAM usage " 81 | "{memory_usage:0.3f} MiB") 82 | output = output_template.format( 83 | time_delta=self.time_delta, 84 | cmd=cmd, 85 | memory_delta=self.memory_delta, 86 | peaked_memory_usage=self.peaked_memory_usage, 87 | memory_usage=new_memory_usage) 88 | print(str(output)) 89 | self.previous_call_memory_usage = new_memory_usage 90 | 91 | 92 | def during_execution_memory_sampler(self): 93 | import time 94 | import memory_profiler 95 | self.peak_memory_usage = -1 96 | self.keep_watching = True 97 | 98 | n = 0 99 | WAIT_BETWEEN_SAMPLES_SECS = 0.001 100 | MAX_ITERATIONS = 60.0 / WAIT_BETWEEN_SAMPLES_SECS 101 | while True: 102 | mem_usage = memory_profiler.memory_usage()[0] 103 | self.peak_memory_usage = max(mem_usage, self.peak_memory_usage) 104 | time.sleep(WAIT_BETWEEN_SAMPLES_SECS) 105 | if not self.keep_watching or n > MAX_ITERATIONS: 106 | # exit if we've been told our command has finished or 107 | # if it has run for more than a sane amount of time 108 | # (e.g. maybe something crashed and we don't want this 109 | # to carry on running) 110 | if n > MAX_ITERATIONS: 111 | print("{} SOMETHING WEIRD HAPPENED AND THIS RAN FOR TOO LONG, THIS THREAD IS KILLING ITSELF".format(__file__)) 112 | break 113 | n += 1 114 | 115 | 116 | def pre_run_cell(self): 117 | """Capture current time before we execute the current command""" 118 | # start a thread that samples RAM usage until the current 119 | # command finishes 120 | ipython_memory_usage_thread = threading.Thread( 121 | target=self.during_execution_memory_sampler) 122 | ipython_memory_usage_thread.daemon = True 123 | ipython_memory_usage_thread.start() 124 | self.t1 = time.time() 125 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ipython_memwatcher: display memory usage during IPython execution 3 | 4 | ipython_memwatcher is an IPython tool to report memory usage deltas for 5 | every command you type. 6 | 7 | This is strongly based on 8 | https://github.com/ianozsvald/ipython_memory_usage by Ian Ozsvald and in 9 | the future ipython_memwatcher can merged back into ipython_memory_usage. 10 | """ 11 | 12 | doclines = __doc__.split("\n") 13 | 14 | # Chosen from http://www.python.org/pypi?:action=list_classifiers 15 | classifiers = """\ 16 | Development Status :: 5 - Production/Stable 17 | Environment :: Console 18 | Intended Audience :: Developers 19 | License :: Free To Use But Restricted 20 | Natural Language :: English 21 | Operating System :: OS Independent 22 | Programming Language :: Python 23 | Topic :: Software Development :: Libraries :: Python Modules 24 | Topic :: Software Development :: Testing 25 | """ 26 | 27 | from setuptools import setup, find_packages 28 | setup( 29 | name="ipython_memwatcher", 30 | version="0.2.5", 31 | url="https://github.com/FrancescAlted/ipython_memwatcher", 32 | author="Francesc Alted, Ian Ozsvald", 33 | author_email="faltet@gmail.com", 34 | maintainer="Francesc Alted", 35 | maintainer_email="faltet@gmail.com", 36 | description=doclines[0], 37 | long_description="\n".join(doclines[2:]), 38 | classifiers=filter(None, classifiers.split("\n")), 39 | platforms=["Any."], 40 | packages=['ipython_memwatcher'], 41 | #package_dir={'ipython_memwatcher': 'ipython_memwatcher'}, 42 | install_requires=['IPython>=3.1', 'memory_profiler'], 43 | ) 44 | --------------------------------------------------------------------------------