├── .coveragerc ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── images ├── analyzing_detached.png ├── debug_detached.png ├── example_output.png ├── inferior_detached.png ├── thread_debugging_enabled.png └── threads.png ├── memory_analyzer ├── __init__.py ├── analysis_utils.py ├── frontend │ ├── __init__.py │ ├── frontend_utils.py │ └── memanz_curses.py ├── gdb_commands.py ├── integrationtest.py ├── memory_analyzer.py ├── templates │ └── analysis.py.template └── tests │ ├── __init__.py │ ├── __main__.py │ ├── test_analysis_template.py │ ├── test_analysis_utils.py │ ├── test_curses.py │ ├── test_frontend.py │ ├── test_gdb_commands.py │ └── test_main_lib.py ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = memory_analyzer 4 | 5 | [report] 6 | precision = 2 7 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: memory_analyzer_tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Running python ${{ matrix.python-version }} on ${{matrix.os}} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | python-version: [3.6, 3.7, 3.8] 12 | os: [macOS-latest, ubuntu-latest] 13 | 14 | steps: 15 | - uses: actions/checkout@v1 16 | 17 | - name: Set up Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | 22 | - name: Update pip + setuptools 23 | run: | 24 | python -m pip install --upgrade pip setuptools coverage 25 | 26 | - name: Install latest version of memory_analyzer 27 | run: | 28 | python -m pip install . 29 | 30 | - name: Run Unitests 31 | run: | 32 | coverage run -m memory_analyzer.tests 33 | 34 | - name: Show test coverage 35 | run: | 36 | coverage report 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .coverage 3 | .eggs 4 | .venv 5 | *.egg-info 6 | __pycache__ 7 | htmlcov 8 | dist 9 | build 10 | .tox 11 | memory_analyzer_out 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct) so that you can understand what actions will and will not be tolerated. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to memory-analyzer 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `main`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## License 30 | By contributing to memory-analyzer, you agree that your contributions will be licensed 31 | under the LICENSE file in the root directory of this source tree. 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Facebook, Inc. and its affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements*.txt LICENSE 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON?=python3 2 | 3 | .PHONY: venv 4 | venv: 5 | $(PYTHON) -m venv .venv 6 | @echo 'run `source .venv/bin/activate` to use virtualenv' 7 | 8 | # The rest of these are intended to be run within the venv, where python points 9 | # to whatever was used to set up the venv. 10 | # 11 | .PHONY: setup 12 | setup: 13 | python3 -m pip install -Ur requirements.txt 14 | python3 -m pip install -Ur requirements-dev.txt 15 | 16 | .PHONY: test 17 | test: 18 | python3 -m coverage run -m memory_analyzer.tests 19 | python3 -m coverage report --omit='.venv/*,.tox/*' --show-missing 20 | 21 | .PHONY: format 22 | format: 23 | @/bin/bash -c 'die() { echo "$$1"; exit 1; }; \ 24 | while read filename; do \ 25 | grep -q "Copyright (c) Facebook" "$$filename" || \ 26 | die "Missing copyright in $$filename"; \ 27 | grep -q "#!/usr/bin/env python3" "$$filename" || \ 28 | die "Missing #! in $$filename"; \ 29 | done < <( git ls-tree -r --name-only HEAD | grep ".py$$" )' 30 | isort --recursive -y memory_analyzer setup.py 31 | black memory_analyzer setup.py 32 | 33 | .PHONY: release 34 | release: 35 | pip install -U wheel 36 | rm -r dist 37 | python3 setup.py sdist bdist_wheel 38 | twine upload dist/* 39 | 40 | # Run this via 'tox -e integration' -- this verifies that templates are 41 | # packaged, and that nothing is leaking through from the repo. 42 | .PHONY: integrationtest 43 | integrationtest: 44 | which memory_analyzer 45 | python3 -m unittest memory_analyzer.integrationtest 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python3 Memory Analyzer For Running Processes 2 | 3 | [![Actions Status](https://github.com/facebookincubator/memory-analyzer/workflows/memory_analyzer_tests/badge.svg)](https://github.com/facebookincubator/memory-analyzer/actions) 4 | 5 | Chasing down a memory leak? Don't want to add debugging code, or even stop your process? 6 | 7 | You've come to the right place. 8 | 9 | # memory_analyzer 10 | 11 | Running the memory analyzer does not require you to stop your process or add any 12 | special debugging code or flags. Running the analysis will not (*cough* should not) break 13 | your process, although your process (and all of its threads!) will be paused while 14 | the memory analyzer gathers information about the objects in memory. 15 | 16 | You will need to install objgraph and pympler in the target binary's library 17 | path, in addition to the deps in `requirements*.txt` for running the frontend. 18 | 19 | ## License 20 | 21 | This source code is licensed under the MIT license. Please see the LICENSE in the root directory for more information. 22 | 23 | ## Things you can find out: 24 | 25 | 1. How many of each object you have in memory 26 | 27 | 2. The total size of all of these objects 28 | 29 | 3. The forward references to the objects 30 | 31 | 4. The backwards references to the objects 32 | 33 | 5. The difference in size/number of objects over time (through snapshot comparisons) 34 | 35 | ### Example Output: 36 | 37 | ![Example Output Screenshot](images/example_output.png) 38 | 39 | ## How To Run 40 | 41 | 1. Find the PID of your process: 42 | 43 | 44 | ps aux | grep 45 | 46 | 47 | 2. Run the memory analyzer against it: 48 | 49 | memory_analyzer run $PID 50 | 51 | This will put you into an ncurses UI and make a binary output file located in `memory_analyzer_out/memory_analyzer_snapshot-{TIMESTAMP}` 52 | 53 | If you are analyzing a process that was run as root, you need to run the analyzer as root as well. 54 | 55 | ### Running on a modern ptrace-limited system 56 | 57 | Modern versions of at least Ubuntu and Arch have a patch that disallows ptracing 58 | of non-child processes by non-root users. 59 | 60 | You can disable this using 61 | 62 | ``` 63 | echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope 64 | ``` 65 | 66 | or run `memory_analyzer` as root. 67 | 68 | 69 | ## View the Output without Re-Analyzing 70 | 71 | If you exit out of the UI but want to view the snapshot data again without re-running the analysis, you can use the 'view' command. 72 | 73 | 74 | memory_analyzer view 75 | 76 | 77 | ## Analyze Multiple Processes 78 | 79 | You can analyze multiple processes at once by simply providing a list of the PIDs separated by spaces. 80 | 81 | memory_analyzer run 1234 4567 890 82 | 83 | If `--snapshot` is used, it tries to pair up the listed PIDs with the PIDs in the snapshot file. If they do pair, a new page is created for each like PID comparing the old and the new version. If it can't find any PIDs that pair up it just compares the first new and first old object. 84 | 85 | If the references flags are used the references are found for all of the PIDs listed. 86 | 87 | ## Viewing Forwards and Backwards References 88 | 89 | You can either view the top N number of objects (sorted by size of object), or you 90 | can look for specific objects. The memory analyzer will generate PNG files displaying the graph, and will upload them for you to phabricator for easy viewing. If you don't want them uploaded you can use the `—no-upload` flag. 91 | 92 | 93 | WARNING: Getting the references is a costly operation! The more references you grab 94 | the longer your process will remain paused for. 95 | 96 | ### View the Top 2 Objects 97 | 98 | memory_analyzer run -s 2 $PID 99 | 100 | ### View specific objects 101 | 102 | To view a specific object you must know it's fully qualified domain name. If you are unsure, I'd recommend running the analyzer first with no flags and identifying the names of the objects you are most interested in there. 103 | 104 | memory_analyzer run $PID -ss __main__.Foo -ss builtins.str 105 | 106 | This will get the forwards and backwards references of the Foo and str objects. 107 | 108 | ## Comparing Snapshots 109 | 110 | Comparing snapshots can be useful to show you how objects grow over time. To create an initial snapshot, just run the analyzer normally: 111 | 112 | memory_analyzer run $PID --snapshot 113 | 114 | By default your snapshot files will all be saved in memory_analyzer_out/. 115 | The snapshot analysis will be located on the second page of the ncurses UI, so hit the arrow key to the right to scroll to the snapshot page once you are in the UI. 116 | 117 | 118 | ## Specify the executable 119 | 120 | The memory analyzer launches GDB with the executable found in `sys.executable`. This might not be the executable you want to use to analyze your binary. For example, you may need to use the debuginfo binary. You can specify the executable with the `-e` flag: 121 | 122 | memory_analyzer run $PID -e /usr/local/bin/python3.7-debuginfo 123 | 124 | ## Custom output file 125 | 126 | If you don't want the file to be the default name, pass in your own custom name. 127 | 128 | 129 | memory_analyzer run $PID -f test_output1 130 | 131 | 132 | 133 | 134 | ## Navigating the UI 135 | 136 | Navigating the UI should be relatively intuitive. You can use the arrow keys or wasd to scroll up and down. 137 | 138 | Scroll Up: Up arrow key or 'w' 139 | 140 | Scroll Down: Down arrow key or 's' 141 | 142 | Switch to next page: Right arrow key or 'd' 143 | 144 | Switch to previous page: Left arrow key or 'a' 145 | 146 | Page Up: Page Up key 147 | 148 | Page Down: Page Down Key 149 | 150 | Jump to Bottom of page: G 151 | 152 | Jump to Top of page: H 153 | 154 | Exit: 'q' 155 | 156 | 157 | 158 | ## Dealing with Multithreading 159 | 160 | Right now the program does NOT run in non-stop mode, meaning that if you have multiple threads they will all be paused during analysis. This could be a problem for some services that require threads to be completing live in the background. It might be possible to support non-stop mode (where only one thread gets paused), but it will only work if the process is stopped and started again by the memory analyzer (this is a limitation of GDB). Please contact me (lisroach) if you are interested in this feature. 161 | 162 | Importantly, this means debugging multithreaded processes goes **much** slower than single threaded- because it takes the time to find and pause each individual thread. If your process has timeouts that could be triggered by a paused thread (e.g. something that talks over the network), use the memory_analyzer with caution. 163 | 164 | 165 | ## FAQ 166 | 167 | ### No data returned 168 | 169 | Try running the process with the `--debug` flag enabled for more information. If there are no obvious errors occurring, your program is likely hung and cannot be analyzed. 170 | 171 | 172 | ### No symbol table is loaded. Use the "file" command. 173 | 174 | This is an error from GDB itself, and it could mean a couple of things: 175 | 1. You don't have the correct debuginfo installed, or 176 | 2. You are using the wrong Python executable. 177 | 178 | To fix the incorrect debuginfo install the debuginfo associated with the python runtime the analyzed process is using. For example: 179 | 180 | sudo yum install python3-debuginfo 181 | 182 | or 183 | sudo apt-get install python3-dbg 184 | 185 | 186 | To solve the wrong Python executable, figure out what executable you need (which python version the analyzed process is running with), and use the --exec flag to specify it: 187 | 188 | memory_analyzer run $PID -e /usr/local/bin/python3.6 189 | 190 | 191 | ### It is hanging 192 | 193 | Try running the memory_analyzer with the `--debug` flag. This is going to show you what GDB is currently doing. 194 | 195 | You should not see: 196 | 197 | `$5 = ` 198 | 199 | or 200 | 201 | ![Inferior Detached](images/inferior_detached.png) 202 | 203 | These are problems. Please file a bug. The memory_analyzer is a new tool, so not all scenarios have been tested and you may have run into something new. Keep reading for more information on understanding the debug output. 204 | 205 | 206 | If you see: 207 | 208 | ![GDB Stopping Many Threads](images/threads.png) 209 | 210 | That means your process has a lot of threads, and it's probably taking GDB a long time to pause them all. 211 | 212 | After this you will see: 213 | 214 | ![Thread Debugging Finished](images/thread_debugging_enabled.png) 215 | 216 | That means GDB is still working on doing GDB things, loading symbols and what not. This is usually what takes the longest, and as of right now I am not sure how to fix it. If you were to run `gdb` directly and attach to your process, this would take just as long :( 217 | 218 | It is not until you see: 219 | 220 | ![Detached](images/debug_detached.png) 221 | 222 | That the analyzer itself is running. Generally this runs very fast and you should barely see these messages flicker by. 223 | 224 | 225 | If you don't see the above, but you do see: 226 | 227 | ![Analyzing and Detached](images/analyzing_detached.png) 228 | 229 | This usually means you are not analyzing the correct process. Perhaps your process is not Python (maybe you analyzed while buck is building)? 230 | 231 | 232 | ### Does it run on Windows 233 | 234 | No. If you are interested in having this available on Windows it should be possible to add this, let `lisroach` know. 235 | 236 | ### Does it work with Python 2 237 | 238 | No, unfortunately this is Python 3 exclusive and is likely to stay that way. 239 | 240 | 241 | # Development 242 | 243 | Any help is welcome and appreciated! 244 | -------------------------------------------------------------------------------- /images/analyzing_detached.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/analyzing_detached.png -------------------------------------------------------------------------------- /images/debug_detached.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/debug_detached.png -------------------------------------------------------------------------------- /images/example_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/example_output.png -------------------------------------------------------------------------------- /images/inferior_detached.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/inferior_detached.png -------------------------------------------------------------------------------- /images/thread_debugging_enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/thread_debugging_enabled.png -------------------------------------------------------------------------------- /images/threads.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookarchive/memory-analyzer/9704810d994704761181eafcb816122751ef0fb7/images/threads.png -------------------------------------------------------------------------------- /memory_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | -------------------------------------------------------------------------------- /memory_analyzer/analysis_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import errno 8 | import io 9 | import os 10 | import pickle 11 | import select 12 | import stat 13 | import subprocess 14 | import sys 15 | from contextlib import contextmanager 16 | from shutil import copyfile 17 | from typing import List 18 | 19 | from attr import dataclass 20 | from jinja2 import Environment, FileSystemLoader 21 | from pympler import summary 22 | 23 | from .frontend import frontend_utils 24 | 25 | 26 | @dataclass 27 | class RetrievedObjects: 28 | pid: int 29 | title: str 30 | data: List[List[int]] 31 | 32 | 33 | class GDBObject: 34 | def __init__(self, pid, current_path, executable, template_out_path): 35 | """ 36 | Args: 37 | 38 | pid: numeric pid of the target 39 | current_path: the directory containing gdb_commands.py 40 | executable: the binary passed as the first arg to gdb 41 | template_out_path: the location that analysis.py rendered templates 42 | end up in. 43 | """ 44 | self.pid = pid 45 | self.fifo = f"/tmp/memanz_pipe_{self.pid}" 46 | self.current_path = current_path 47 | # These should all be the same, so safe for threads. 48 | os.putenv("MEMORY_ANALYZER_TEMPLATES_PATH", template_out_path) 49 | self.executable = executable 50 | 51 | def run_analysis(self, debug=False): 52 | self.create_pipe() 53 | frontend_utils.echo_info(f"Analyzing pid {self.pid}") 54 | command_file = f"{self.current_path}/gdb_commands.py" 55 | command = [ 56 | "gdb", 57 | "-q", 58 | # Activates python for GDB. 59 | self.executable, 60 | "-p", 61 | f"{self.pid}", 62 | "-ex", 63 | "set trace-commands on", 64 | f"{'-batch' if debug else '-batch-silent'}", 65 | # This shouldn't be required since we specify absolute path, but 66 | # TODO this gives us a way to inject a path with objgraph on it. 67 | "-ex", 68 | # Lets gdb find the correct gdb_commands script. 69 | f"set directories {self.current_path}", 70 | "-ex", 71 | # Sets the correct path for gdb_commands, else C-API commands fail. 72 | f'py sys.path.append("{self.current_path}")', 73 | "-x", 74 | f"{command_file}", 75 | ] 76 | frontend_utils.echo_info(f"Setting up GDB for pid {self.pid}") 77 | proc = subprocess.Popen( 78 | command, stderr=sys.stderr if debug else subprocess.DEVNULL 79 | ) 80 | with self.drain_pipe(proc) as data: 81 | retrieved_objs = RetrievedObjects( 82 | pid=self.pid, 83 | title=f"Analysis for {self.pid}", 84 | data=self.unpickle_pipe(data), 85 | ) 86 | 87 | self._end_subprocess(proc) 88 | return retrieved_objs 89 | 90 | @contextmanager 91 | def drain_pipe(self, process): 92 | """ 93 | We need this because by default, `open`s on named pipes block. If GDB or 94 | the injected GDB extension in Python crash, the process will never write 95 | to the pipe and we will block opening and `memory_analyzer` won't exit. 96 | """ 97 | try: 98 | pipe = os.open(self.fifo, os.O_RDONLY | os.O_NONBLOCK) 99 | result = io.BytesIO() 100 | timeout = 0.1 # seconds 101 | 102 | partial_read = None 103 | while bool(partial_read) or process.poll() is None: 104 | ready_fds, _, _ = select.select([pipe], [], [], timeout) 105 | 106 | if len(ready_fds) > 0: 107 | ready_fd = ready_fds[0] 108 | try: 109 | partial_read = os.read(ready_fd, select.PIPE_BUF) 110 | except BlockingIOError: 111 | partial_read = None 112 | 113 | if partial_read: 114 | result.write(partial_read) 115 | 116 | result.seek(0) 117 | yield result 118 | result.close() 119 | except Exception as e: 120 | frontend_utils.echo_error(f"Failed with {e}") 121 | self._end_subprocess(process) 122 | sys.exit(1) 123 | finally: 124 | os.close(pipe) 125 | 126 | def _end_subprocess(self, proc): 127 | try: 128 | proc.wait(5) 129 | except TimeoutError: 130 | proc.kill() 131 | 132 | def create_pipe(self): 133 | try: 134 | os.mkfifo(self.fifo) 135 | os.chmod(str(self.fifo), 0o666) 136 | except OSError as oe: 137 | if oe.errno != errno.EEXIST: 138 | raise 139 | 140 | def unpickle_pipe(self, fifo): 141 | frontend_utils.echo_info("Gathering data...") 142 | try: 143 | items = pickle.load(fifo) 144 | if items: 145 | if isinstance(items, Exception): 146 | raise items 147 | return items 148 | except EOFError: 149 | return 150 | except pickle.UnpicklingError as e: 151 | frontend_utils.echo_error(f"Error retrieving data from process: {e}") 152 | raise 153 | except Exception as e: 154 | frontend_utils.echo_error( 155 | f"{type(e).__name__} occurred during analysis: {e}" 156 | ) 157 | raise 158 | 159 | 160 | def load_template(name, templates_path): 161 | env = Environment(autoescape=False, loader=FileSystemLoader(templates_path)) 162 | return env.get_template(name) 163 | 164 | 165 | def render_template( 166 | template_name, 167 | templates_path, 168 | num_refs, 169 | pid, 170 | specific_refs, 171 | output_path, 172 | template_out_dir, 173 | ): 174 | objgraph_template = load_template(template_name, templates_path) 175 | template = objgraph_template.render( 176 | num_refs=num_refs, pid=pid, specific_refs=specific_refs, output_path=output_path 177 | ) 178 | # This path has to match the end of gdb_commands.py; the env var is set in 179 | # GDBObject constructor above. 180 | if template_out_dir: 181 | with open( 182 | os.path.join(template_out_dir, f"rendered_template-{pid}.py.out"), "w" 183 | ) as f: 184 | f.write(template) 185 | return template 186 | 187 | 188 | def snapshot_diff(cur_items, snapshot_file): 189 | """ 190 | Attempts to compare like PIDs. If like PIDS can't be found it will just compare 191 | the first PID listed to the first PID in the file. Any unmatched or non-first 192 | PIDs will be ignored because we don't know what to compare them to. 193 | """ 194 | try: 195 | prev_items = list(frontend_utils.get_pages(snapshot_file)) 196 | except pickle.UnpicklingError as e: 197 | frontend_utils.echo_error( 198 | f"Error unpickling the data from {snapshot_file}: {e}" 199 | ) 200 | return None 201 | 202 | differences = [] 203 | for cur_item in cur_items: 204 | for prev_item in prev_items: 205 | if cur_item.pid == prev_item.pid: 206 | diff = summary.get_diff(cur_item.data, prev_item.data) 207 | differences.append( 208 | RetrievedObjects( 209 | pid=cur_item.pid, 210 | title=f"Snapshot Differences for {cur_item.pid}", 211 | data=diff, 212 | ) 213 | ) 214 | if not differences: 215 | diff = summary.get_diff(cur_items[0].data, prev_items[0].data) 216 | differences.append( 217 | RetrievedObjects(pid=0, title=f"Snapshot Differences", data=diff) 218 | ) 219 | return differences 220 | -------------------------------------------------------------------------------- /memory_analyzer/frontend/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | -------------------------------------------------------------------------------- /memory_analyzer/frontend/frontend_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import curses 8 | import os 9 | import pickle 10 | import subprocess 11 | 12 | import click 13 | import prettytable 14 | 15 | from . import memanz_curses 16 | 17 | 18 | def readable_size(i, snapshot=False): 19 | """ 20 | Pretty-print the integer `i` as a human-readable size representation. 21 | """ 22 | degree = 0 23 | while i > 1024: 24 | i = i / float(1024) 25 | degree += 1 26 | scales = ["B", "KB", "MB", "GB", "TB", "EB"] 27 | if snapshot: 28 | return f"{i:+.2f}{scales[degree]:>5}" 29 | return f"{i:.2f}{scales[degree]:>5}" 30 | 31 | 32 | def init_table(references, snapshot): 33 | pt = prettytable.PrettyTable() 34 | field_names = ["Object", "Count", "Size"] 35 | if references: 36 | field_names.extend(["References", "Backwards References"]) 37 | if snapshot: 38 | field_names[1] += " Diff" 39 | field_names[2] += " Diff" 40 | pt.field_names = field_names 41 | pt.align["Object"] = "l" 42 | pt.align[pt.field_names[1]] = "r" 43 | pt.align[pt.field_names[2]] = "r" 44 | return pt 45 | 46 | 47 | def format_summary_output(page): 48 | """ 49 | Formats in prettytable style the pympler summary. 50 | """ 51 | references = False 52 | items = page.data 53 | if not items: 54 | items = [[f"No data to display for pid {page.pid}.", 0, 0]] 55 | if any(len(item) == 5 for item in items): 56 | references = True 57 | snapshot = "Snapshot Differences" in page.title 58 | pt = init_table(references, snapshot) 59 | items.sort(key=lambda x: x[2], reverse=True) 60 | for sublist in items: 61 | if snapshot: 62 | sublist[1] = f"{sublist[1]:+}" 63 | sublist[2] = readable_size(sublist[2], snapshot) 64 | if len(sublist) != len(pt.field_names): 65 | # Fill in missing data with "". 66 | sublist.extend(["" for _ in range(len(pt.field_names) - len(sublist))]) 67 | pt.add_row(sublist) 68 | return pt 69 | 70 | 71 | def table_as_list_of_strings(table): 72 | string_table = table.get_string() 73 | return string_table.split("\n") 74 | 75 | 76 | def get_pages(filename): 77 | """ 78 | Read each pickled object in the given file as a page. 79 | The objects will be lists of lists. 80 | """ 81 | with open(filename, "rb") as fd: 82 | while True: 83 | try: 84 | yield pickle.load(fd) 85 | except EOFError: 86 | break 87 | except pickle.UnpicklingError: 88 | raise 89 | 90 | 91 | def view(stdscr, pages): 92 | """ 93 | Format the data in a list of pretty tables that will be the pages of the 94 | curses UI, then activate the nCurses UI. 95 | """ 96 | pages_as_tables = [] 97 | titles = [] 98 | for page in pages: 99 | titles.append(page.title) 100 | table = format_summary_output(page) 101 | pages_as_tables.append(table_as_list_of_strings(table)) 102 | if pages_as_tables: 103 | win = memanz_curses.Window(stdscr, pages_as_tables, titles) 104 | win.run() 105 | 106 | 107 | def initiate_curses(items): 108 | curses.wrapper(view, items) 109 | 110 | 111 | def echo_error(msg, *args, **kwargs): 112 | click.echo(click.style(f"ERROR: {msg}", fg="red"), *args, **kwargs) 113 | 114 | 115 | def echo_info(msg, *args, **kwargs): 116 | click.echo(click.style(f"{msg}", fg="green"), *args, **kwargs) 117 | -------------------------------------------------------------------------------- /memory_analyzer/frontend/memanz_curses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | """ 8 | This can either be run standalone with an already generated file (so the user 9 | does not have to run the memory analyzer multiple times), or it can be launched 10 | directly by the memory analyzer. 11 | """ 12 | 13 | import curses 14 | from collections import namedtuple 15 | 16 | UP_KEYS = [curses.KEY_UP, ord("w"), ord("k"), ord("H"), curses.KEY_PPAGE] 17 | DOWN_KEYS = [curses.KEY_DOWN, ord("s"), ord("j"), ord("G"), curses.KEY_NPAGE] 18 | RIGHT_KEYS = [curses.KEY_RIGHT, ord("d"), ord("l")] 19 | LEFT_KEYS = [curses.KEY_LEFT, ord("a"), ord("h")] 20 | 21 | 22 | class Window: 23 | UP = -1 24 | DOWN = 1 25 | 26 | def __init__(self, stdscr, pages, titles): 27 | self.top = 0 28 | self.position = self.top 29 | self.pages = pages 30 | self.page_titles = titles 31 | self.cur_page = 0 32 | self.window = stdscr 33 | self.height = curses.LINES - 1 34 | self.width = curses.COLS - 1 35 | self.bottom = len(self.cur_page.items) 36 | curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE) 37 | 38 | @property 39 | def cur_page(self): 40 | return self._cur_page 41 | 42 | @cur_page.setter 43 | def cur_page(self, pos): 44 | page = namedtuple("Page", "pos items title") 45 | self._cur_page = page( 46 | pos=pos, items=self.pages[pos], title=self.page_titles[pos] 47 | ) 48 | return self._cur_page 49 | 50 | def status_bar_render(self): 51 | statusbarstr = ( 52 | f"{self.cur_page.title} | Navigate with arrows or wasd | Press 'q' to exit" 53 | ) 54 | self.window.attron(curses.color_pair(1)) 55 | self.window.addstr(self.height, 0, statusbarstr) 56 | self.window.addstr( 57 | self.height, len(statusbarstr), " " * (self.width - len(statusbarstr)) 58 | ) 59 | self.window.attroff(curses.color_pair(1)) 60 | 61 | def run(self): 62 | self.display() 63 | self.user_input() 64 | 65 | def display(self): 66 | self.window.erase() 67 | for idx, item in enumerate( 68 | self.cur_page.items[self.position : self.position + self.height - 1] 69 | ): 70 | self.window.addstr(idx, 0, item) 71 | self.status_bar_render() 72 | self.window.refresh() 73 | 74 | def scroll_up(self, user_select): 75 | if ( 76 | user_select in [curses.KEY_UP, ord("w"), ord("k")] 77 | and self.position != self.top 78 | ): 79 | self.position += self.UP 80 | elif user_select == ord("H") and self.position != self.top: 81 | self.position = self.top 82 | elif user_select == curses.KEY_PPAGE and self.position != self.top: 83 | if self.position - self.height > self.top: 84 | self.position = self.position - self.height 85 | else: 86 | self.position = self.top 87 | 88 | def scroll_down(self, user_select): 89 | if ( 90 | user_select in [curses.KEY_DOWN, ord("s"), ord("j")] 91 | and self.position + self.height <= self.bottom 92 | ): 93 | self.position += self.DOWN 94 | elif user_select == ord("G") and self.position + self.height != self.bottom: 95 | self.position = self.bottom - self.height + 1 96 | elif user_select == curses.KEY_NPAGE: 97 | if self.position + self.height < self.bottom - self.height: 98 | self.position = self.position + self.height 99 | else: 100 | self.position = self.bottom - self.height + 1 101 | 102 | def scroll_right(self, user_select): 103 | if len(self.pages) != self.cur_page.pos + 1: 104 | self.cur_page = self.cur_page.pos + 1 105 | self.bottom = len(self.cur_page.items) 106 | 107 | def scroll_left(self, user_select): 108 | if self.cur_page.pos != 0: 109 | self.cur_page = self.cur_page.pos - 1 110 | self.bottom = len(self.cur_page.items) 111 | 112 | def user_input(self): 113 | user_select = self.window.getch() 114 | while user_select != ord("q"): 115 | self.status_bar_render() 116 | if self.bottom > self.height: 117 | if user_select in UP_KEYS: 118 | self.scroll_up(user_select) 119 | elif user_select in DOWN_KEYS: 120 | self.scroll_down(user_select) 121 | if user_select in RIGHT_KEYS: 122 | self.scroll_right(user_select) 123 | elif user_select in LEFT_KEYS: 124 | self.scroll_left(user_select) 125 | 126 | self.display() 127 | user_select = self.window.getch() 128 | -------------------------------------------------------------------------------- /memory_analyzer/gdb_commands.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | """Module containing all of the custom GDB commands to be used for memory analysis. 7 | This is equivalent to a GDB command file, and can only be called from a GDB process.""" 8 | 9 | import os 10 | import sys 11 | 12 | import gdb 13 | 14 | TEMPLATES_PATH = os.getenv("MEMORY_ANALYZER_TEMPLATES_PATH") 15 | 16 | 17 | def lock_GIL(func): 18 | def wrapper(*args): 19 | out = gdb.execute("call (void*) PyGILState_Ensure()", to_string=True) 20 | gil_value = next((x for x in out.split() if x.startswith("$")), "$1") 21 | print("GIL", gil_value) 22 | func(*args) 23 | call = "call (void) PyGILState_Release(" + gil_value + ")" 24 | gdb.execute(call) 25 | 26 | return wrapper 27 | 28 | 29 | class FileCommand(gdb.Command): 30 | def __init__(self): 31 | super(FileCommand, self).__init__("file_command", gdb.COMMAND_NONE) 32 | 33 | @lock_GIL 34 | def invoke(self, filename, from_tty): 35 | cmd_string = "with open('{filename}') as f: exec(f.read())".format( 36 | filename=filename 37 | ) 38 | gdb.execute( 39 | 'call (void) PyRun_SimpleString("{cmd_str}")'.format(cmd_str=cmd_string) 40 | ) 41 | 42 | 43 | FileCommand() 44 | pid = gdb.selected_inferior().pid 45 | gdb.execute( 46 | "file_command {TEMPLATES_PATH}/rendered_template-{pid}.py.out".format( 47 | TEMPLATES_PATH=TEMPLATES_PATH, pid=pid 48 | ) 49 | ) 50 | -------------------------------------------------------------------------------- /memory_analyzer/integrationtest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import subprocess 9 | import sys 10 | import tempfile 11 | import time 12 | import unittest 13 | 14 | 15 | class IntegrationTest(unittest.TestCase): 16 | def test_works_at_all(self): 17 | output_name = tempfile.mktemp() 18 | print(output_name) 19 | 20 | # This tells us that everything important was packaged if we tox 21 | # installed an sdist, but doesn't tell us anything if this was setup.py 22 | # develop'd in a git repo. 23 | os.chdir("/") 24 | 25 | self.assertFalse(os.path.exists(output_name)) 26 | with open("/proc/sys/kernel/yama/ptrace_scope") as f: 27 | value = f.read().strip() 28 | 29 | self.assertEqual("0", value, "/proc/sys/kernel/yama/ptrace_scope should be 0") 30 | 31 | # Presumably this is a virtualenv python executable that has objgraph 32 | # and pympler 33 | try: 34 | child = subprocess.Popen( 35 | [sys.executable, "-c", "import sys; sys.stdin.readline()"], 36 | stdin=subprocess.PIPE, 37 | ) 38 | # TODO figure out how we can ensure setup is done; right now we're 39 | # just relying on it taking a while to launch/attach 40 | analyzer = subprocess.Popen( 41 | ["memory_analyzer", "run", "-q", "-f", output_name, str(child.pid)] 42 | ) 43 | rc = analyzer.wait(5) 44 | finally: 45 | child.communicate(b"\n") 46 | 47 | self.assertEqual(0, rc) 48 | self.assertTrue(os.path.exists(output_name)) 49 | # TODO verify pickle has some strs 50 | 51 | 52 | if __name__ == "__main__": 53 | unittest.main() 54 | -------------------------------------------------------------------------------- /memory_analyzer/memory_analyzer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import errno 8 | import os 9 | import pickle 10 | import sys 11 | import tempfile 12 | from datetime import datetime 13 | from functools import partial 14 | from multiprocessing.pool import ThreadPool 15 | 16 | import click 17 | import pkg_resources 18 | 19 | from . import analysis_utils 20 | from .frontend import frontend_utils 21 | 22 | 23 | def analyze_memory_launcher( 24 | pid, num_refs, specific_refs, debug, output_file, executable, template_out_path 25 | ): 26 | templates_path = ( 27 | pkg_resources.resource_filename("memory_analyzer", "templates") + "/" 28 | ) 29 | cur_path = os.path.dirname(__file__) + "/" # not zip safe, for now 30 | gdb_obj = analysis_utils.GDBObject(pid, cur_path, executable, template_out_path) 31 | output_path = os.path.abspath(output_file) 32 | analysis_utils.render_template( 33 | f"analysis.py.template", 34 | templates_path, 35 | num_refs, 36 | pid, 37 | specific_refs, 38 | output_path, 39 | template_out_path, 40 | ) 41 | return gdb_obj.run_analysis(debug) 42 | 43 | 44 | def write_to_output_file(filename, items): 45 | with open(filename, "wb+") as outputf: 46 | for item in items: 47 | bytes_item = pickle.dumps(item) 48 | outputf.write(bytes_item) 49 | 50 | 51 | def is_root(): 52 | if os.geteuid() == 0: 53 | return True 54 | return False 55 | 56 | 57 | def validate_pids(ctx, param, pids): 58 | for pid in pids: 59 | pid = int(pid) 60 | try: 61 | os.kill(pid, 0) 62 | except OSError as e: 63 | if e.errno == errno.EPERM and not is_root(): 64 | msg = "Permission error, try running as root" 65 | raise click.UsageError(msg) 66 | 67 | msg = f"The given PID {pid} is not valid." 68 | raise click.BadParameter(msg) 69 | 70 | return pids 71 | 72 | 73 | def check_positive_int(ctx, param, i): 74 | i = int(i) 75 | if i >= 0: 76 | return i 77 | msg = "The number of references cannot be negative." 78 | raise click.BadParameter(msg) 79 | 80 | 81 | @click.group() 82 | def cli(): 83 | pass 84 | 85 | 86 | @cli.command() 87 | @click.argument("filename", type=click.Path(exists=True)) 88 | def view(filename): 89 | """ 90 | Tool for viewing the output of the memory analyzer. Launches a UI. 91 | 92 | Argument: 93 | 94 | FILENAME: The filename of the snapshot to view. 95 | """ 96 | try: 97 | pages = frontend_utils.get_pages(filename) 98 | except pickle.UnpicklingError as e: 99 | frontend_utils.echo_error(f"Error unpickling the data from {filename}: {e}") 100 | sys.exit(1) 101 | frontend_utils.initiate_curses(pages) 102 | 103 | 104 | @cli.command() 105 | @click.argument("pids", callback=validate_pids, nargs=-1) 106 | @click.option( 107 | "-s", 108 | "--show-references", 109 | "num_refs", 110 | default=0, 111 | callback=check_positive_int, 112 | help="Shows the references of the X most common objects.\n\ 113 | This is a costly operation, do not use a large number.", 114 | ) 115 | @click.option( 116 | "-ss", 117 | "--show-specific-references", 118 | "specific_refs", 119 | multiple=True, 120 | default=[], 121 | help="Shows the references of all objects given.\n\ 122 | This is a costly operation, be careful.", 123 | ) 124 | @click.option( 125 | "--snapshot", 126 | type=click.Path(exists=True), 127 | help="The file containing snapshot information of previous run.", 128 | ) 129 | @click.option( 130 | "-q", 131 | "--quiet", 132 | "quiet", 133 | is_flag=True, 134 | default=False, 135 | help="Don't enter UI after evaluation.", 136 | ) 137 | @click.option( 138 | "-d", 139 | "--debug", 140 | "debug", 141 | is_flag=True, 142 | default=False, 143 | help="Show GDB output, for debugging the analyzer.", 144 | ) 145 | @click.option("-f", "--output-file", type=str, help="File to output results to.") 146 | @click.option( 147 | "--no-upload", 148 | is_flag=True, 149 | default=False, 150 | help="Do not upload reference graphs to phabricator.", 151 | ) 152 | @click.option( 153 | "-e", 154 | "--exec", 155 | "executable", 156 | help="Python executable to use", 157 | default=f"{sys.executable}-dbg", 158 | ) 159 | def run( 160 | pids, 161 | num_refs, 162 | specific_refs, 163 | snapshot, 164 | quiet, 165 | debug, 166 | output_file, 167 | no_upload, 168 | executable, 169 | ): 170 | """ 171 | Tool for providing memory analysis on a running Python3 process. 172 | 173 | Argument: 174 | 175 | PIDS: The pid or list of pids of the running Python 3 process(es) to evaluate. 176 | 177 | 178 | Output: 179 | 180 | A binary file of the results. By default, after run completes the user 181 | will enter a UI for navigating the data. If references are set, png 182 | files will also be created and uploaded to phabricator. 183 | 184 | 185 | Unless otherwise set, the output files will reside in memory_analyzer_out/, 186 | which is created where the service is ran. 187 | """ 188 | runtime = "{:%Y%m%d%H%M%S}".format(datetime.now()) 189 | default_filename = f"memory_analyzer_out/memory_analyzer_snapshot-{runtime}" 190 | if not output_file: 191 | output_file = default_filename 192 | 193 | references = num_refs > 0 or specific_refs 194 | retrieved_objs = [] 195 | # Create a folder for output 196 | if references or output_file == default_filename: 197 | os.makedirs(os.path.dirname(default_filename), exist_ok=True) 198 | template_out_path = tempfile.mkdtemp() 199 | 200 | # Make a new thread per PID 201 | worker_pool = ThreadPool(len(pids)) 202 | target = partial( 203 | analyze_memory_launcher, 204 | num_refs=num_refs, 205 | specific_refs=specific_refs, 206 | debug=debug, 207 | output_file=output_file, 208 | executable=executable, 209 | template_out_path=template_out_path, 210 | ) 211 | 212 | for result in worker_pool.imap_unordered(target, pids): 213 | if result.data is None: 214 | frontend_utils.echo_error( 215 | f"{result.title} returned no data! Try rerunning with --debug" 216 | ) 217 | else: 218 | retrieved_objs.append(result) 219 | if not retrieved_objs: 220 | frontend_utils.echo_error("No results to report") 221 | sys.exit(1) 222 | if snapshot: 223 | diffs = analysis_utils.snapshot_diff(retrieved_objs, snapshot) 224 | retrieved_objs.extend(diffs) 225 | 226 | frontend_utils.echo_info(f"Writing output to file {output_file}") 227 | write_to_output_file(output_file, retrieved_objs) 228 | if not quiet: 229 | frontend_utils.echo_info("Initializing frontend...") 230 | frontend_utils.initiate_curses(retrieved_objs) 231 | 232 | 233 | if __name__ == "__main__": 234 | cli() 235 | -------------------------------------------------------------------------------- /memory_analyzer/templates/analysis.py.template: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | try: 7 | import objgraph 8 | import os 9 | import sys 10 | import re 11 | from pympler import muppy, summary 12 | import pickle 13 | try: 14 | from types import InstanceType 15 | except ImportError: 16 | # Python 3.x 17 | InstanceType = None 18 | 19 | 20 | class SwallowPrints: 21 | def __enter__(self): 22 | self._old_stdout = sys.stdout 23 | sys.stdout = open(os.devnull, 'w') 24 | 25 | def __exit__(self, exc_type, exc_val, exc_tb): 26 | sys.stdout.close() 27 | sys.stdout = self._old_stdout 28 | 29 | def _get_obj_type(obj): 30 | objtype = type(obj) 31 | if type(obj) == InstanceType: 32 | objtype = obj.__class__ 33 | return objtype 34 | 35 | def _repr(obj): 36 | objtype = _get_obj_type(obj) 37 | name = objtype.__name__ 38 | module = getattr(objtype, '__module__', None) 39 | if module: 40 | return f"{module}.{name}" 41 | else: 42 | return name 43 | 44 | summary._repr = _repr 45 | 46 | def forward_references(dirname, obj, shortname): 47 | filename_forw = f'{dirname}/ref_{{ pid }}_{shortname}.png' 48 | objgraph.show_refs(obj, filename=filename_forw) 49 | return filename_forw 50 | 51 | def backwards_references(dirname, obj, shortname): 52 | filename_back = f'{dirname}/backref_{{ pid }}_{shortname}.png' 53 | objgraph.show_backrefs(obj, filename=filename_back) 54 | return filename_back 55 | 56 | 57 | def create_references(sublist): 58 | obj = sublist[0] 59 | obj_shortname = obj.split('.')[-1] 60 | obj_shortname = re.sub("[^A-Za-z0-9._]+", "", obj_shortname) 61 | dirname = os.path.dirname("{{ output_path }}") 62 | with SwallowPrints(): 63 | forw = forward_references(dirname, obj, obj_shortname) 64 | back = backwards_references(dirname, obj, obj_shortname) 65 | return forw, back 66 | 67 | all_objects = muppy.get_objects(remove_dups=True) 68 | summ = summary.summarize(all_objects) 69 | if {{ num_refs }} > 0: 70 | summ.sort(key=lambda x: x[2], reverse=True) 71 | for sublist in summ[:{{ num_refs }}]: 72 | forw, back = create_references(sublist) 73 | sublist.extend([forw, back]) 74 | for obj in {{ specific_refs }}: 75 | for sublist in summ: 76 | if obj in sublist[0]: 77 | forw, back = create_references(sublist) 78 | sublist.extend([forw, back]) 79 | break 80 | 81 | with open('/tmp/memanz_pipe_{{ pid }}', 'wb') as fifo: 82 | pickle.dump(summ, fifo) 83 | 84 | except Exception as e: 85 | print("Got exception", e) 86 | import pickle 87 | # We don't want exceptions here to affect the profiled process but want to 88 | # see the exceptions in the UI 89 | with open('/tmp/memanz_pipe_{{ pid }}', 'wb') as fifo: 90 | pickle.dump(e, fifo) 91 | -------------------------------------------------------------------------------- /memory_analyzer/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from .test_analysis_template import ObjGraphTemplateTests 8 | from .test_analysis_utils import AnalysisUtilsTest 9 | from .test_curses import MemanzCursesTest 10 | from .test_frontend import FrontendUtilsTest 11 | from .test_gdb_commands import GdbCommandsTests 12 | from .test_main_lib import FakeOSError, MainLibTests 13 | -------------------------------------------------------------------------------- /memory_analyzer/tests/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import unittest 8 | 9 | if __name__ == "__main__": 10 | unittest.main(module="memory_analyzer.tests", verbosity=2) 11 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_analysis_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import pickle 9 | import sys 10 | import tempfile 11 | from unittest import TestCase, mock 12 | 13 | import objgraph 14 | from jinja2 import Environment, FileSystemLoader 15 | from pympler import muppy, summary # noqa 16 | 17 | from .. import analysis_utils 18 | 19 | 20 | class ObjGraphTemplateTests(TestCase): 21 | template_name = "analysis.py.template" 22 | filename = "some_filename" 23 | templates_path = f"{os.path.abspath(os.path.dirname(__file__))}/../templates/" 24 | pid = 1234 25 | specific_refs = ["str", "int"] 26 | 27 | def setUp(self): 28 | self.items = [ 29 | ["builtins.test1", 1, 10000], 30 | ["__main__.test2", 3, 5], 31 | ["ast._things.test3", 10, 1024], 32 | ] 33 | # TODO: Add tests for summary obj and the _repr 34 | mock_summ = mock.patch.object(summary, "summarize", return_value=self.items) 35 | mock_summ.start() 36 | self.addCleanup(mock_summ.stop) 37 | 38 | def tearDown(self): 39 | if os.path.isfile(f"{self.templates_path}rendered_template.py.out"): 40 | os.remove(f"{self.templates_path}rendered_template.py.out") 41 | 42 | def test_with_no_references(self): 43 | template = analysis_utils.render_template( 44 | self.template_name, 45 | self.templates_path, 46 | 0, 47 | self.pid, 48 | [], 49 | self.filename, 50 | None, 51 | ) 52 | with mock.patch("builtins.open", mock.mock_open(), create=True) as mock_fifo: 53 | exec(template) 54 | mock_fifo.assert_called_with(f"/tmp/memanz_pipe_{self.pid}", "wb") 55 | output_bytes = pickle.dumps(self.items) 56 | mock_fifo().write.assert_called_with(output_bytes) 57 | 58 | @mock.patch.object(objgraph, "show_backrefs") 59 | @mock.patch.object(objgraph, "show_refs") 60 | def test_with_num_references(self, mock_refs, mock_back_refs): 61 | dirname = "/tmp/tests/" 62 | template = analysis_utils.render_template( 63 | self.template_name, 64 | self.templates_path, 65 | 1, 66 | self.pid, 67 | [], 68 | f"{dirname}{self.filename}", 69 | None, 70 | ) 71 | with mock.patch("builtins.open", mock.mock_open(), create=True) as mock_fifo: 72 | exec(template, {}) 73 | handler = mock_fifo() 74 | output_bytes = pickle.dumps(self.items) 75 | handler.write.assert_called_with(output_bytes) 76 | self.assertEqual( 77 | self.items, 78 | [ 79 | [ 80 | "builtins.test1", 81 | 1, 82 | 10000, 83 | f"{dirname}ref_1234_test1.png", 84 | f"{dirname}backref_1234_test1.png", 85 | ], 86 | ["ast._things.test3", 10, 1024], 87 | ["__main__.test2", 3, 5], 88 | ], 89 | ) 90 | 91 | @mock.patch.object(objgraph, "show_backrefs") 92 | @mock.patch.object(objgraph, "show_refs") 93 | def test_with_specific_references(self, mock_refs, mock_back_refs): 94 | dirname = "/tmp/tests/" 95 | with tempfile.TemporaryDirectory() as d: 96 | template = analysis_utils.render_template( 97 | self.template_name, 98 | self.templates_path, 99 | 0, 100 | self.pid, 101 | ["test3"], 102 | f"{dirname}{self.filename}", 103 | d, 104 | ) 105 | self.assertEqual(1, len(os.listdir(d)), os.listdir(d)) 106 | 107 | with mock.patch("builtins.open", mock.mock_open(), create=True) as mock_fifo: 108 | exec(template, {}) 109 | handler = mock_fifo() 110 | output_bytes = pickle.dumps(self.items) 111 | handler.write.assert_called_with(output_bytes) 112 | self.assertEqual( 113 | self.items, 114 | [ 115 | ["builtins.test1", 1, 10000], 116 | ["__main__.test2", 3, 5], 117 | [ 118 | "ast._things.test3", 119 | 10, 120 | 1024, 121 | f"{dirname}ref_1234_test3.png", 122 | f"{dirname}backref_1234_test3.png", 123 | ], 124 | ], 125 | ) 126 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_analysis_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import pickle 9 | import subprocess 10 | import sys 11 | from unittest import TestCase, mock 12 | 13 | from .. import analysis_utils 14 | 15 | 16 | class AnalysisUtilsTest(TestCase): 17 | PID = 123 18 | CURRENT_PATH = os.path.abspath(f"{os.path.dirname(__file__)}/..") 19 | 20 | def setUp(self): 21 | self.gdb = analysis_utils.GDBObject( 22 | self.PID, self.CURRENT_PATH, sys.executable, "/tmp" 23 | ) 24 | self.filepath = os.path.abspath( 25 | f"{os.path.dirname(__file__)}/../gdb_commands.py" 26 | ) 27 | self.executable = sys.executable 28 | # Swallow the info messages 29 | patch_info = mock.patch("memory_analyzer.frontend.frontend_utils.echo_info") 30 | self.mock_info = patch_info.start() 31 | self.addCleanup(self.mock_info.stop) 32 | 33 | @mock.patch("memory_analyzer.analysis_utils.copyfile") 34 | @mock.patch("memory_analyzer.analysis_utils.subprocess.Popen", autospec=True) 35 | def test_command_string_built_correctly(self, mock_sub, _): 36 | with mock.patch("builtins.open", mock.mock_open()): 37 | self.gdb.unpickle_pipe = mock.MagicMock() 38 | self.gdb.run_analysis() 39 | cmd_list = [ 40 | "gdb", 41 | "-q", 42 | self.executable, 43 | "-p", 44 | f"{self.PID}", 45 | "-ex", 46 | "set trace-commands on", 47 | "-batch-silent", 48 | "-ex", 49 | f"set directories {self.CURRENT_PATH}", 50 | "-ex", 51 | f'py sys.path.append("{self.CURRENT_PATH}")', 52 | "-x", 53 | f"{self.filepath}", 54 | ] 55 | mock_sub.assert_called_with(cmd_list, stderr=subprocess.DEVNULL) 56 | calls = [ 57 | mock.call(f"Analyzing pid {self.PID}"), 58 | mock.call(f"Setting up GDB for pid {self.PID}"), 59 | ] 60 | self.mock_info.assert_has_calls(calls) 61 | 62 | @mock.patch("memory_analyzer.analysis_utils.copyfile") 63 | @mock.patch("memory_analyzer.analysis_utils.subprocess.Popen", autospec=True) 64 | def test_command_string_built_correctly_debug_mode(self, mock_sub, _): 65 | with mock.patch("builtins.open", mock.mock_open()): 66 | self.gdb.unpickle_pipe = mock.MagicMock() 67 | self.gdb.run_analysis(debug=True) 68 | cmd_list = [ 69 | "gdb", 70 | "-q", 71 | self.executable, 72 | "-p", 73 | f"{self.PID}", 74 | "-ex", 75 | "set trace-commands on", 76 | "-batch", 77 | "-ex", 78 | f"set directories {self.CURRENT_PATH}", 79 | "-ex", 80 | f'py sys.path.append("{self.CURRENT_PATH}")', 81 | "-x", 82 | f"{self.filepath}", 83 | ] 84 | mock_sub.assert_called_with(cmd_list, stderr=sys.stderr) 85 | calls = [ 86 | mock.call(f"Analyzing pid {self.PID}"), 87 | mock.call(f"Setting up GDB for pid {self.PID}"), 88 | ] 89 | self.mock_info.assert_has_calls(calls) 90 | 91 | @mock.patch("memory_analyzer.analysis_utils.pickle.load") 92 | @mock.patch("memory_analyzer.frontend.frontend_utils.echo_error") 93 | def test_unpickle_pipe_errors(self, mock_echo, mock_pickle): 94 | mock_pickle.side_effect = NameError("Random Exception") 95 | with self.assertRaises(NameError): 96 | self.gdb.unpickle_pipe("Fifo Data") 97 | mock_echo.assert_called_with( 98 | "NameError occurred during analysis: Random Exception" 99 | ) 100 | 101 | @mock.patch("memory_analyzer.analysis_utils.pickle.load") 102 | @mock.patch("memory_analyzer.frontend.frontend_utils.echo_error") 103 | def test_unpickle_pipe_unpickle_errors(self, mock_echo, mock_pickle): 104 | mock_pickle.side_effect = pickle.UnpicklingError("Error") 105 | with self.assertRaises(pickle.UnpicklingError): 106 | self.gdb.unpickle_pipe("Fifo Data") 107 | mock_echo.assert_called_with("Error retrieving data from process: Error") 108 | 109 | @mock.patch("memory_analyzer.analysis_utils.pickle.load") 110 | @mock.patch("memory_analyzer.frontend.frontend_utils.echo_error") 111 | def test_read_items_exception(self, mock_echo, mock_pickle): 112 | mock_pickle.return_value = AttributeError("Whats up") 113 | with self.assertRaises(AttributeError): 114 | self.gdb.unpickle_pipe("Fifo Data") 115 | mock_echo.assert_called_with( 116 | "AttributeError occurred during analysis: Whats up" 117 | ) 118 | 119 | @mock.patch("memory_analyzer.analysis_utils.pickle.load") 120 | @mock.patch("memory_analyzer.frontend.frontend_utils.echo_error") 121 | def test_snapshot_diff_error(self, mock_echo, mock_pickle): 122 | mock_pickle.side_effect = pickle.UnpicklingError("Error") 123 | with mock.patch("builtins.open", mock.mock_open()): 124 | out = analysis_utils.snapshot_diff(["Item"], "filename") 125 | mock_echo.assert_called_with("Error unpickling the data from filename: Error") 126 | self.assertIsNone(out) 127 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_curses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import curses 8 | from unittest import TestCase, mock 9 | 10 | from ..frontend import memanz_curses 11 | 12 | 13 | class MemanzCursesTest(TestCase): 14 | def setUp(self): 15 | self.mock_curses = mock.patch( 16 | "memory_analyzer.frontend.memanz_curses.curses" 17 | ).start() 18 | self.addCleanup(self.mock_curses.stop) 19 | self.mock_curses.LINES = 2 20 | self.mock_curses.COLS = 100 21 | self.mock_curses.KEY_DOWN = curses.KEY_DOWN 22 | self.mock_curses.KEY_UP = curses.KEY_UP 23 | self.mock_curses.KEY_PPAGE = curses.KEY_PPAGE 24 | self.mock_curses.KEY_NPAGE = curses.KEY_NPAGE 25 | self.mock_curses.KEY_RIGHT = curses.KEY_RIGHT 26 | self.mock_curses.KEY_LEFT = curses.KEY_LEFT 27 | self.statusbarstr = " | Navigate with arrows or wasd | Press 'q' to exit" 28 | self.pages = [["Page1", 10, 1024], ["Page2", 90, 100]] 29 | self.titles = ["Analysis of 1234", "Snapshot Differences"] 30 | self.win = memanz_curses.Window(self.mock_curses, self.pages, self.titles) 31 | 32 | def test_status_bar_render(self): 33 | self.win.status_bar_render() 34 | 35 | def test_user_input_scroll_up_and_down_wasd(self): 36 | self.win.window.getch.side_effect = [ord("s"), ord("q")] 37 | self.win.user_input() 38 | self.assertEqual(self.win.position, 1) 39 | self.win.window.getch.side_effect = [ord("w"), ord("q")] 40 | self.win.user_input() 41 | self.assertEqual(self.win.position, 0) 42 | 43 | def test_user_input_scroll_up_and_down_hjkl(self): 44 | self.win.window.getch.side_effect = [ord("j"), ord("q")] 45 | self.win.user_input() 46 | self.assertEqual(self.win.position, 1) 47 | self.win.window.getch.side_effect = [ord("k"), ord("q")] 48 | self.win.user_input() 49 | self.assertEqual(self.win.position, 0) 50 | 51 | def test_user_input_scroll_up_and_down_arrows(self): 52 | self.win.window.getch.side_effect = [self.mock_curses.KEY_DOWN, ord("q")] 53 | self.win.user_input() 54 | self.assertEqual(self.win.position, 1) 55 | self.win.window.getch.side_effect = [self.mock_curses.KEY_UP, ord("q")] 56 | self.win.user_input() 57 | self.assertEqual(self.win.position, 0) 58 | 59 | def test_user_input_attempt_to_scroll_up_off_window(self): 60 | self.assertEqual(self.win.position, 0) 61 | self.win.window.getch.side_effect = [self.mock_curses.KEY_UP, ord("q")] 62 | self.win.user_input() 63 | self.assertEqual(self.win.position, 0) 64 | self.win.window.getch.side_effect = [self.mock_curses.KEY_PPAGE, ord("q")] 65 | self.win.user_input() 66 | self.assertEqual(self.win.position, 0) 67 | 68 | def test_user_input_attempt_to_scroll_down_off_window(self): 69 | self.win.position = self.win.bottom 70 | self.win.window.getch.side_effect = [self.mock_curses.KEY_DOWN, ord("q")] 71 | self.win.user_input() 72 | self.assertEqual(self.win.position, self.win.bottom) 73 | self.win.window.getch.side_effect = [self.mock_curses.KEY_NPAGE, ord("q")] 74 | self.win.user_input() 75 | self.assertEqual(self.win.position, self.win.bottom) 76 | 77 | def test_page_left_and_right_arrows(self): 78 | self.win.window.getch.side_effect = [self.mock_curses.KEY_RIGHT, ord("q")] 79 | self.win.user_input() 80 | self.assertEqual(self.win.cur_page.pos, 1) 81 | self.assertEqual(self.win.cur_page.items, self.pages[1]) 82 | self.win.window.getch.side_effect = [self.mock_curses.KEY_LEFT, ord("q")] 83 | self.win.user_input() 84 | self.assertEqual(self.win.cur_page.pos, 0) 85 | self.assertEqual(self.win.cur_page.items, self.pages[0]) 86 | 87 | def test_page_left_and_right_wasd(self): 88 | self.win.window.getch.side_effect = [ord("d"), ord("q")] 89 | self.win.user_input() 90 | self.assertEqual(self.win.cur_page.pos, 1) 91 | self.assertEqual(self.win.cur_page.items, self.pages[1]) 92 | self.win.window.getch.side_effect = [ord("a"), ord("q")] 93 | self.win.user_input() 94 | self.assertEqual(self.win.cur_page.pos, 0) 95 | self.assertEqual(self.win.cur_page.items, self.pages[0]) 96 | 97 | def test_page_left_and_right_hjkl(self): 98 | self.win.window.getch.side_effect = [ord("l"), ord("q")] 99 | self.win.user_input() 100 | self.assertEqual(self.win.cur_page.pos, 1) 101 | self.assertEqual(self.win.cur_page.items, self.pages[1]) 102 | self.win.window.getch.side_effect = [ord("h"), ord("q")] 103 | self.win.user_input() 104 | self.assertEqual(self.win.cur_page.pos, 0) 105 | self.assertEqual(self.win.cur_page.items, self.pages[0]) 106 | 107 | def test_display_normal(self): 108 | self.mock_curses.LINES = 10 109 | win = memanz_curses.Window(self.mock_curses, self.pages, self.titles) 110 | win.position = 1 111 | win.display() 112 | win.window.addstr.assert_any_call(0, 0, 10) 113 | win.window.addstr.assert_any_call(1, 0, 1024) 114 | win.window.addstr.assert_any_call(9, 0, self.titles[0] + self.statusbarstr) 115 | 116 | def test_display_window_too_small_for_display(self): 117 | self.win.position = 1 118 | self.win.display() 119 | calls = [ 120 | mock.call( 121 | self.mock_curses.LINES - 1, 0, self.titles[0] + self.statusbarstr 122 | ), 123 | mock.call( 124 | self.mock_curses.LINES - 1, 125 | len(self.titles[0] + self.statusbarstr), 126 | " " 127 | * (self.mock_curses.COLS - 1 - len(self.titles[0] + self.statusbarstr)), 128 | ), 129 | ] 130 | self.win.window.addstr.assert_has_calls(calls) 131 | 132 | def test_display_window_size_zero(self): 133 | self.mock_curses.LINES = 0 134 | win = memanz_curses.Window(self.mock_curses, self.pages, self.titles) 135 | win.position = 1 136 | win.display() 137 | win.window.addstr.assert_any_call(0, 0, 10) 138 | win.window.addstr.assert_any_call(-1, 0, self.titles[0] + self.statusbarstr) 139 | 140 | def test_set_cur_page(self): 141 | self.assertEqual(self.win.cur_page.items, self.pages[0]) 142 | self.assertEqual(self.win.cur_page.pos, 0) 143 | self.win.cur_page = 1 144 | self.assertEqual(self.win.cur_page.items, self.pages[1]) 145 | self.assertEqual(self.win.cur_page.pos, 1) 146 | 147 | def test_display_snapshot_page(self): 148 | self.mock_curses.LINES = 10 149 | win = memanz_curses.Window(self.mock_curses, self.pages, self.titles) 150 | win.cur_page = 1 151 | self.assertEqual(["Page2", 90, 100], win.cur_page.items) 152 | win.display() 153 | win.window.addstr.assert_any_call(0, 0, "Page2") 154 | win.window.addstr.assert_any_call(1, 0, 90) 155 | win.window.addstr.assert_any_call(2, 0, 100) 156 | win.window.addstr.assert_any_call( 157 | 9, 0, "Snapshot Differences" + self.statusbarstr 158 | ) 159 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_frontend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from tempfile import NamedTemporaryFile 8 | from unittest import TestCase, mock 9 | 10 | from .. import analysis_utils 11 | from ..frontend import frontend_utils 12 | 13 | 14 | class FrontendUtilsTest(TestCase): 15 | def test_readable_size_correct_no_snapshot(self): 16 | scales = ["B", "KB", "MB", "GB", "TB", "EB"] 17 | original_val = 10 18 | for i in range(6): 19 | val = original_val * (1024 ** i) 20 | string_val = frontend_utils.readable_size(val) 21 | correct_output = f"10.00{scales[i]:>5}" 22 | self.assertEqual(string_val, correct_output) 23 | 24 | def test_readable_size_correct_with_snapshot(self): 25 | scales = ["B", "KB", "MB", "GB", "TB", "EB"] 26 | original_val = 10 27 | for i in range(6): 28 | val = original_val * (1024 ** i) 29 | string_val = frontend_utils.readable_size(val, True) 30 | correct_output = f"+10.00{scales[i]:>5}" 31 | self.assertEqual(string_val, correct_output) 32 | 33 | def test_readable_size_neg_input(self): 34 | original_val = -10 35 | string_val = frontend_utils.readable_size(original_val) 36 | correct_output = "-10.00 B" 37 | self.assertEqual(string_val, correct_output) 38 | 39 | def test_readable_size_invalid_input(self): 40 | original_val = "10" 41 | with self.assertRaises(TypeError): 42 | frontend_utils.readable_size(original_val) 43 | 44 | def test_init_table_field_names(self): 45 | pt = frontend_utils.init_table(references=False, snapshot=False) 46 | self.assertEqual(pt.field_names, ["Object", "Count", "Size"]) 47 | pt = frontend_utils.init_table(references=False, snapshot=True) 48 | self.assertEqual(pt.field_names, ["Object", "Count Diff", "Size Diff"]) 49 | pt = frontend_utils.init_table(references=True, snapshot=False) 50 | self.assertEqual( 51 | pt.field_names, 52 | ["Object", "Count", "Size", "References", "Backwards References"], 53 | ) 54 | pt = frontend_utils.init_table(references=True, snapshot=True) 55 | self.assertEqual( 56 | pt.field_names, 57 | ["Object", "Count Diff", "Size Diff", "References", "Backwards References"], 58 | ) 59 | 60 | def test_init_table_alignment(self): 61 | pt = frontend_utils.init_table(references=False, snapshot=False) 62 | self.assertEqual(pt._align, {"Object": "l", "Count": "r", "Size": "r"}) 63 | pt = frontend_utils.init_table(references=True, snapshot=False) 64 | self.assertEqual( 65 | pt._align, 66 | { 67 | "Object": "l", 68 | "Count": "r", 69 | "Size": "r", 70 | "References": "c", 71 | "Backwards References": "c", 72 | }, 73 | ) 74 | pt = frontend_utils.init_table(references=True, snapshot=True) 75 | self.assertEqual( 76 | pt._align, 77 | { 78 | "Object": "l", 79 | "Count Diff": "r", 80 | "Size Diff": "r", 81 | "References": "c", 82 | "Backwards References": "c", 83 | }, 84 | ) 85 | pt = frontend_utils.init_table(references=False, snapshot=True) 86 | self.assertEqual( 87 | pt._align, {"Object": "l", "Count Diff": "r", "Size Diff": "r"} 88 | ) 89 | 90 | def test_format_output_default(self): 91 | items = analysis_utils.RetrievedObjects( 92 | pid=1234, 93 | title="Analysis of pid 1234", 94 | data=[["Item 1", 10, 1024], ["Item 2", 1000, 1_048_576]], 95 | ) 96 | correct_items = [ 97 | ["Item 2", 1000, "1024.00 KB"], 98 | ["Item 1", 10, "1024.00 B"], 99 | ] 100 | pt = frontend_utils.format_summary_output(items) 101 | self.assertEqual(pt._rows, correct_items) 102 | 103 | def test_format_output_no_data(self): 104 | items = analysis_utils.RetrievedObjects( 105 | pid=1234, title="Analysis of pid 1234", data=None 106 | ) 107 | correct_items = [[f"No data to display for pid 1234.", 0, "0.00 B"]] 108 | pt = frontend_utils.format_summary_output(items) 109 | self.assertEqual(pt._rows, correct_items) 110 | 111 | def test_format_output_snapshot(self): 112 | items = analysis_utils.RetrievedObjects( 113 | pid=1234, 114 | title="Snapshot Differences", 115 | data=[["Item 1", 10, 1024], ["Item 2", 1000, 1_048_576]], 116 | ) 117 | correct_items = [ 118 | ["Item 2", "+1000", "+1024.00 KB"], 119 | ["Item 1", "+10", "+1024.00 B"], 120 | ] 121 | pt = frontend_utils.format_summary_output(items) 122 | self.assertEqual(pt._rows, correct_items) 123 | 124 | def test_format_output_with_references(self): 125 | items = analysis_utils.RetrievedObjects( 126 | pid=1234, 127 | title="Analysis of pid 1234", 128 | data=[ 129 | ["Item 1", 10, 1034, "filename.png", "filename2.png"], 130 | ["Item 2", 1000, 10042], 131 | ], 132 | ) 133 | updated_items = [ 134 | ["Item 2", 1000, frontend_utils.readable_size(10042), "", ""], 135 | [ 136 | "Item 1", 137 | 10, 138 | frontend_utils.readable_size(1034), 139 | "filename.png", 140 | "filename2.png", 141 | ], 142 | ] 143 | pt = frontend_utils.format_summary_output(items) 144 | self.assertEqual(pt._rows, items.data) 145 | self.assertEqual(items.data, updated_items) 146 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_gdb_commands.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import builtins 8 | import imp 9 | import os 10 | import sys 11 | from unittest import TestCase, mock 12 | 13 | # Because we cannot `import gdb` except for modules called via GDB, we must 14 | # mock the `import gdb` in gdb_commands. 15 | mock_gdb = mock.MagicMock() 16 | sys.modules["gdb"] = mock_gdb 17 | 18 | if True: 19 | from .. import gdb_commands # isort: skip doesn't appear to work 20 | 21 | 22 | class GdbCommandsTests(TestCase): 23 | 24 | # def test_alter_path_strings(self): 25 | # func = mock.Mock() 26 | # wrap = gdb_commands.alter_path(func) 27 | # wrap() 28 | # first_str = "import sys; sys.path.append('test_path')" 29 | # call_1 = f'call PyRun_SimpleString("{first_str}")' 30 | # second_str = "sys.path.remove('test_path')" 31 | # call_2 = f'call PyRun_SimpleString("{second_str}")' 32 | # mock_calls = [mock.call(call_1), mock.call(call_2)] 33 | # mock_gdb.execute.assert_has_calls(mock_calls) 34 | 35 | @mock.patch("sys.stdout.write") 36 | def test_lock_GIL(self, mock_write): 37 | func = mock.Mock() 38 | mock_gdb.execute.return_value = ( 39 | "[New Thread 0x7f48e4dd7700 (LWP 2640572)]\n$2 = PyGILState_UNLOCKED\n" 40 | ) 41 | wrap = gdb_commands.lock_GIL(func) 42 | wrap() 43 | call_1 = "call (void*) PyGILState_Ensure()" 44 | call_2 = "call (void) PyGILState_Release($2)" 45 | mock_calls = [mock.call(call_1, to_string=True), mock.call(call_2)] 46 | mock_gdb.execute.assert_has_calls(mock_calls) 47 | mock_write.assert_has_calls([mock.call("$2"), mock.call("\n")]) 48 | 49 | @mock.patch("sys.stdout.write") 50 | def test_lock_GIL_weird_return(self, mock_write): 51 | func = mock.Mock() 52 | mock_gdb.execute.return_value = "[New Thread 0x7f48e4dd7700 (LWP 2640572)]" 53 | wrap = gdb_commands.lock_GIL(func) 54 | wrap() 55 | call_1 = "call (void*) PyGILState_Ensure()" 56 | call_2 = "call (void) PyGILState_Release($1)" 57 | mock_calls = [mock.call(call_1, to_string=True), mock.call(call_2)] 58 | mock_gdb.execute.assert_has_calls(mock_calls) 59 | mock_write.assert_has_calls([mock.call("$1"), mock.call("\n")]) 60 | -------------------------------------------------------------------------------- /memory_analyzer/tests/test_main_lib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import errno 8 | from functools import partial 9 | from unittest import TestCase, mock 10 | 11 | import click 12 | 13 | from memory_analyzer import memory_analyzer 14 | 15 | 16 | class FakeOSError(OSError): 17 | def __init__(self, errno): 18 | super().__init__() 19 | self.errno = errno 20 | 21 | 22 | def fake_kill(pid, sig, errno=None): 23 | if pid != 42: 24 | raise FakeOSError(errno) 25 | 26 | 27 | class MainLibTests(TestCase): 28 | @mock.patch("memory_analyzer.memory_analyzer.os.geteuid") 29 | def test_is_root_passes(self, mock_geteuid): 30 | mock_geteuid.return_value = 0 31 | self.assertTrue(memory_analyzer.is_root()) 32 | 33 | @mock.patch("memory_analyzer.memory_analyzer.os.geteuid") 34 | def test_is_root_fails(self, mock_geteuid): 35 | mock_geteuid.return_value = 42 36 | self.assertFalse(memory_analyzer.is_root()) 37 | 38 | @mock.patch("memory_analyzer.memory_analyzer.os.kill") 39 | def test_validate_pids_with_valid_pids(self, mock_kill): 40 | ctx = param = mock.MagicMock() 41 | mock_kill.side_effect = fake_kill 42 | self.assertEqual([42], memory_analyzer.validate_pids(ctx, param, [42])) 43 | 44 | @mock.patch("memory_analyzer.memory_analyzer.os.geteuid") 45 | @mock.patch("memory_analyzer.memory_analyzer.os.kill") 46 | def test_validate_pids_kills_with_signal_value_zero(self, mock_kill, mock_geteuid): 47 | mock_geteuid.return_value = 42 48 | ctx = param = mock.MagicMock() 49 | 50 | memory_analyzer.validate_pids(ctx, param, [42, 314]) 51 | 52 | mock_kill.assert_has_calls([mock.call(42, 0), mock.call(314, 0)]) 53 | 54 | @mock.patch("memory_analyzer.memory_analyzer.os.geteuid") 55 | @mock.patch("memory_analyzer.memory_analyzer.os.kill") 56 | def test_validate_pids_with_an_invalid_pid_and_no_root( 57 | self, mock_kill, mock_geteuid 58 | ): 59 | mock_geteuid.return_value = 42 60 | ctx = param = mock.MagicMock() 61 | mock_kill.side_effect = partial(fake_kill, errno=errno.EPERM) 62 | 63 | with self.assertRaises(click.UsageError): 64 | memory_analyzer.validate_pids(ctx, param, [42, 314]) 65 | 66 | @mock.patch("memory_analyzer.memory_analyzer.os.geteuid") 67 | @mock.patch("memory_analyzer.memory_analyzer.os.kill") 68 | def test_validate_pids_with_an_invalid_pid_and_error_is_not_permission_related( 69 | self, mock_kill, mock_geteuid 70 | ): 71 | mock_geteuid.return_value = 42 72 | ctx = param = mock.MagicMock() 73 | mock_kill.side_effect = partial(fake_kill, errno=errno.ESRCH) 74 | 75 | with self.assertRaises(click.BadParameter): 76 | memory_analyzer.validate_pids(ctx, param, [314]) 77 | 78 | def test_check_positive_int_valid(self): 79 | ctx = param = mock.MagicMock() 80 | 81 | self.assertEqual(0, memory_analyzer.check_positive_int(ctx, param, 0)) 82 | self.assertEqual(42, memory_analyzer.check_positive_int(ctx, param, 42)) 83 | 84 | def test_check_positive_int_invalid(self): 85 | ctx = param = mock.MagicMock() 86 | 87 | with self.assertRaises(click.BadParameter): 88 | memory_analyzer.check_positive_int(ctx, param, -1) 89 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | isort 3 | black 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | attrs 3 | jinja2 4 | prettytable 5 | pympler 6 | objgraph 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tox:tox] 2 | envlist = py36, py37, py38, integration 3 | 4 | [testenv] 5 | deps = -rrequirements-dev.txt 6 | whitelist_externals = make 7 | commands = 8 | make test 9 | setenv = 10 | py{36,37,38}: COVERAGE_FILE={envdir}/.coverage 11 | 12 | [testenv:integration] 13 | commands = 14 | make integrationtest 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) Facebook, Inc. and its affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | from setuptools import find_packages, setup 9 | 10 | with open("README.md") as f: 11 | readme = f.read() 12 | 13 | with open("requirements.txt") as f: 14 | requires = f.read().strip().splitlines() 15 | 16 | setup( 17 | name="memory_analyzer", 18 | description="Python 3 memory analyzer for running processes", 19 | long_description=readme, 20 | long_description_content_type="text/markdown", 21 | version="0.1.2", 22 | author="Lisa Roach, Facebook", 23 | author_email="lisroach@fb.com", 24 | url="https://github.com/facebookincubator/memory-analyzer", 25 | classifiers=[ 26 | "Development Status :: 1 - Planning", 27 | "Intended Audience :: Developers", 28 | "License :: OSI Approved :: MIT License", 29 | "Programming Language :: Python", 30 | "Programming Language :: Python :: 3", 31 | "Programming Language :: Python :: 3.6", 32 | "Programming Language :: Python :: 3.7", 33 | ], 34 | license="MIT", 35 | packages=find_packages(), 36 | package_data={"memory_analyzer": ["templates/*.template"]}, 37 | test_suite="memory_analyzer.tests", 38 | python_requires=">=3.6", 39 | setup_requires=["setuptools"], 40 | install_requires=requires, 41 | entry_points={ 42 | "console_scripts": ["memory_analyzer = memory_analyzer.memory_analyzer:cli"] 43 | }, 44 | ) 45 | --------------------------------------------------------------------------------