├── .gitattributes ├── .gitignore ├── Demo.ipynb ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── load_whyprofiler_on_startup.ipy ├── setup.py ├── whyprofiler.gif └── whyprofiler ├── __init__.py └── static ├── index.js ├── semgrep └── use_orjson.yaml ├── semgrep_experiments ├── README.txt └── use_set_not_list.yaml └── whyprofiler.css /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 106 | __pypackages__/ 107 | 108 | # Celery stuff 109 | celerybeat-schedule 110 | celerybeat.pid 111 | 112 | # SageMath parsed files 113 | *.sage.py 114 | 115 | # Environments 116 | .env 117 | .venv 118 | env/ 119 | venv/ 120 | ENV/ 121 | env.bak/ 122 | venv.bak/ 123 | 124 | # Spyder project settings 125 | .spyderproject 126 | .spyproject 127 | 128 | # Rope project settings 129 | .ropeproject 130 | 131 | # mkdocs documentation 132 | /site 133 | 134 | # mypy 135 | .mypy_cache/ 136 | .dmypy.json 137 | dmypy.json 138 | 139 | # Pyre type checker 140 | .pyre/ 141 | 142 | # pytype static type analyzer 143 | .pytype/ 144 | 145 | # Cython debug symbols 146 | cython_debug/ 147 | 148 | # PyCharm 149 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 150 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 151 | # and can be added to the global gitignore or merged into this file. For a more nuclear 152 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 153 | #.idea/ 154 | -------------------------------------------------------------------------------- /Demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "fixed": false, 8 | "previous_time": 0 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "import json\n", 13 | "import sys\n", 14 | "\n", 15 | "def generate_json():\n", 16 | " l = []\n", 17 | " for _ in range(1000000):\n", 18 | " l.append({\"a\" : 101231238931.223432, \"b\" : \"test\", \"c\": [\"abc\"]})\n", 19 | " return l\n", 20 | "\n", 21 | "print(\"This is a tech demo of Whyprofiler - a joint Robusta.dev and R2C.dev project\")\n", 22 | "print(\"Soon we will know which part of this code is slow, but how to fix it?\")\n", 23 | "print(\"Note that applying the fix takes time because it installs orjson with pip\")\n", 24 | "z=generate_json()\n", 25 | "print(\"Still.. running\", flush=True)\n", 26 | "sys.stdout.flush() # without this the print only appears at the end\n", 27 | "x=json.dumps(z)\n", 28 | "print(\"Done!\")\n", 29 | "# we delete these to free up memory\n", 30 | "del z\n", 31 | "del x\n", 32 | "\n", 33 | "# Want to automate your response to Kubernetes alerts and events using Python?\n", 34 | "# Try robusta.dev" 35 | ] 36 | } 37 | ], 38 | "metadata": { 39 | "kernelspec": { 40 | "display_name": "Python 3", 41 | "language": "python", 42 | "name": "python3" 43 | }, 44 | "language_info": { 45 | "codemirror_mode": { 46 | "name": "ipython", 47 | "version": 3 48 | }, 49 | "file_extension": ".py", 50 | "mimetype": "text/x-python", 51 | "name": "python", 52 | "nbconvert_exporter": "python", 53 | "pygments_lexer": "ipython3", 54 | "version": "3.8.8" 55 | } 56 | }, 57 | "nbformat": 4, 58 | "nbformat_minor": 4 59 | } 60 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/datascience-notebook 2 | 3 | WORKDIR /app 4 | 5 | COPY . . 6 | COPY load_whyprofiler_on_startup.ipy /home/jovyan/.ipython/profile_default/startup/load_whyprofiler_on_startup.ipy 7 | USER root 8 | 9 | RUN chown jovyan -R /app 10 | RUN chown jovyan -R /home/jovyan/.ipython 11 | USER jovyan 12 | 13 | RUN python setup.py install --user 14 | RUN jupyter nbextension install --py whyprofiler --user 15 | RUN jupyter nbextension enable --py whyprofiler --user 16 | RUN jupyter serverextension enable --py whyprofiler --user 17 | RUN chmod a+x /home/jovyan/.local/lib/python3.9/site-packages/semgrep-*/semgrep/bin/semgrep-core 18 | ENV DOCKER_STACKS_JUPYTER_CMD notebook 19 | 20 | EXPOSE 8888 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Robusta Dev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include whyprofiler/static *.js 2 | recursive-include whyprofiler/static *.css 3 | recursive-include whyprofiler/static *.yaml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | WhyProfiler is a CPU profiler for Jupyter notebook that not only identifies hotspots but can suggest faster alternatives. It is powered by [Semgrep](https://semgrep.dev/) and [yappi](https://github.com/sumerc/yappi) and implemented as Jupyter Notebook extension. 4 | 5 | It is also the only Python profiler that both identifies hotspots and recommends equivalent code that can fix them. 6 | 7 | ![WhyProfiler in action](./whyprofiler.gif) 8 | 9 | [See the release blog post for more information.](https://home.robusta.dev/blog/whyprofiler-the-worlds-first-hybrid-cpu-profiler-now-for-jupyter-notebook-and-python/) 10 | 11 | # Building and running 12 | 13 | The easiest way to use whyprofiler is to run it as a Docker container. The container includes both Jupyter Notebook as well as the whyprofiler extension for Jupyter notebook, preconfigured. 14 | 15 | ```bash 16 | docker build . -t whyprofiler 17 | docker run -p 8888:8888 whyprofiler 18 | ``` 19 | 20 | We don't yet support M1 Macbooks. If you have an M1 Macbook please build an amd64 version of the image instead: 21 | 22 | ```bash 23 | docker build --platform linux/amd64 . --progress=plain -t whyprofiler 24 | ``` 25 | 26 | # Adding new rules to whyprofiler 27 | Add a new file with a Semgrep rule to ./whyprofiler/static/semgrep/ 28 | 29 | Look at the existing rules for reference. 30 | 31 | Some rules requires installing python packages in order to apply the fix. You can add the `fix_installation` field to Semgrep metadata to do so. [See example.](https://github.com/aantn/nprofile/blob/master/whyprofiler/static/semgrep/use_orjson.yaml#L12) 32 | -------------------------------------------------------------------------------- /load_whyprofiler_on_startup.ipy: -------------------------------------------------------------------------------- 1 | # we can do this here or in each jupyter notebook 2 | # just keep in mind that if this isn't present here then you need to refresh the webbrowser after running this in the notebook 3 | %load_ext whyprofiler 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="whyprofiler", 8 | version="0.0.1", 9 | author="Natan Yellin", 10 | author_email="natan@robusta.dev", 11 | description="A hybrid profiler for Jupyter notebook", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/robusta-dev/whyprofiler", 15 | include_package_data = True, 16 | install_requires=["jupyter", "yappi", "semgrep"], 17 | zip_safe=False, 18 | packages=setuptools.find_packages(), 19 | classifiers=[ 20 | "Programming Language :: Python :: 3", 21 | "License :: OSI Approved :: MIT License", 22 | "Operating System :: OS Independent", 23 | ], 24 | ) -------------------------------------------------------------------------------- /whyprofiler.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robusta-dev/WhyProfiler/a2a352ac0836ee49d1a5a02a58453ec079061995/whyprofiler.gif -------------------------------------------------------------------------------- /whyprofiler/__init__.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | import subprocess 3 | import yappi 4 | import json 5 | import os 6 | import sys 7 | from contextlib import ExitStack 8 | from warnings import warn 9 | 10 | class whyprofiler: 11 | def __init__(self, ip): 12 | self.comm = None 13 | self.ipython = ip 14 | self.installed_packages = set() 15 | 16 | def get_semgrep_command(self, input_file, fix=False, check_id=None): 17 | rules_path = os.path.join(os.path.dirname(__file__), "static", "semgrep") 18 | if check_id is not None: 19 | rules_path = os.path.join(rules_path, check_id + ".yaml") 20 | 21 | cmd = [ 22 | "python", 23 | "-m", 24 | "semgrep", 25 | "-f", rules_path, 26 | "--lang=py", 27 | "--error", 28 | "--json", 29 | "--disable-version-check", 30 | "--no-rewrite-rule-ids", 31 | input_file 32 | ] 33 | 34 | if fix: 35 | cmd.append("--autofix") 36 | 37 | return cmd 38 | 39 | 40 | def run_semgrep(self, code, fix=False, check_id=None): 41 | with ExitStack() as stack: 42 | # don't add directly to the ExitStack because on Windows we need to close the file earlier so that we can write to it 43 | with tempfile.NamedTemporaryFile(delete=False) as code_file: 44 | code_file.write(code.encode()) 45 | stack.callback(lambda: os.remove(code_file.name)) 46 | 47 | semgrep_cmd = self.get_semgrep_command(code_file.name, fix, check_id) 48 | result = subprocess.run(semgrep_cmd, capture_output=True, text=True) 49 | #print(result) 50 | data = json.loads(result.stdout) 51 | 52 | code_file = stack.enter_context(open(code_file.name, "r")) 53 | fixed_code = code_file.read() 54 | return data, fixed_code 55 | 56 | 57 | def apply_fix(self, code, check_id): 58 | data, fixed_code = self.run_semgrep(code, fix=True, check_id=check_id) 59 | requirements = data['results'][0]['extra']['metadata']['fix_installation'] 60 | if requirements not in self.installed_packages: 61 | warn("installing " + requirements) 62 | self.ipython.run_line_magic('pip', f'install {requirements} --user') 63 | self.installed_packages.add(requirements) 64 | return data, fixed_code 65 | 66 | def pre_run_cell(self, info): 67 | if self.comm is None: 68 | return 69 | 70 | yappi.set_clock_type("wall") 71 | yappi.start() 72 | 73 | 74 | def post_run_cell(self, result): 75 | # TODO: capture stdout and stderr so that extension bugs don't leak their way out 76 | #print("post run cell") 77 | if self.comm is None: 78 | return 79 | 80 | yappi.stop() 81 | 82 | semgrep, _ = self.run_semgrep(result.info.raw_cell) 83 | 84 | timing = {} 85 | for stat in yappi.get_func_stats(filter_callback=lambda stat: stat.name == "" and stat.ctx_name == "_MainThread"): 86 | timing[stat.lineno] = stat.ttot 87 | self.comm.send({ 88 | "msg_type" : "analysis_ready", 89 | "line_timing" : timing, 90 | "analysis" : semgrep 91 | }) 92 | yappi.clear_stats() # is there a use for global statistics? 93 | 94 | 95 | def on_comm(self, comm, open_msg): 96 | # comm is the kernel Comm instance 97 | # msg is the comm_open message 98 | # TODO verify that self.comm is None or close old and replace with new? (can multiple commms all connect?) 99 | print("Comm opened with message", open_msg, "and commm", comm) 100 | self.comm = comm 101 | 102 | # Register handler for later messages 103 | @comm.on_msg 104 | def _recv(msg): 105 | content = msg['content']['data'] 106 | # TODO: support multiple comms and use comm ids... 107 | # Use msg['content']['data'] for the data in the message 108 | if content['msg_type'] == "apply_fix": 109 | fix, fixed_code = self.apply_fix(content['code'], content['check_id']) 110 | comm.send({ 111 | 'msg_type': 'fix_ready', 112 | 'request' : content, 113 | 'fix' : fix, 114 | 'fixed_code' : fixed_code 115 | }) 116 | else: 117 | comm.send({ 118 | 'msg_type': 'error_unknown_request', 119 | 'request' : content, 120 | }) 121 | 122 | 123 | def _jupyter_server_extension_paths(): 124 | return [{ 125 | "module": "whyprofiler" 126 | }] 127 | 128 | 129 | # Jupyter Extension points 130 | def _jupyter_nbextension_paths(): 131 | return [dict( 132 | section="notebook", 133 | src="static", 134 | dest="whyprofiler", 135 | require="whyprofiler/index")] 136 | 137 | 138 | def load_jupyter_server_extension(nbapp): 139 | nbapp.log.info("************************ jupyter server extension loaded") 140 | 141 | 142 | def load_ipython_extension(ip): 143 | #print("************************* ipython extension loaded") 144 | ip.np = whyprofiler(ip) 145 | ip.events.register("pre_run_cell", ip.np.pre_run_cell) 146 | ip.events.register("post_run_cell", ip.np.post_run_cell) 147 | ip.kernel.comm_manager.register_target('whyprofiler', ip.np.on_comm) 148 | 149 | 150 | def unload_ipython_extension(ip): 151 | ip.events.unregister("pre_run_cell", ip.np.pre_run_cell) 152 | ip.events.unregister("post_run_cell", ip.np.post_run_cell) 153 | ip.kernel.comm_manager.unregister_target('whyprofiler', ip.np.on_comm) 154 | del ip.np 155 | -------------------------------------------------------------------------------- /whyprofiler/static/index.js: -------------------------------------------------------------------------------- 1 | console.log("whyprofiler javascript loaded") 2 | 3 | /* useful: 4 | var cell = Jupyter.notebook.get_selected_cell(); 5 | */ 6 | 7 | define([ 8 | 'require', 9 | 'jquery', 10 | 'moment', 11 | "underscore", 12 | 'base/js/namespace', 13 | 'base/js/events', 14 | 'notebook/js/codecell' 15 | ], function ( 16 | requirejs, 17 | $, 18 | moment, 19 | _, 20 | Jupyter, 21 | events, 22 | codecell 23 | ) { 24 | 'use strict'; 25 | const THRESHOLD_SECONDS = 1.0; 26 | const THRESHOLD_PERCENTAGE = 0.1; 27 | const GUTTER = "whyprofiler-codemirror-gutter"; 28 | var log_prefix = '[whyprofiler]'; 29 | var comm = null; 30 | var cell; 31 | var line_widgets = []; 32 | console.log(log_prefix, "js initialization"); 33 | 34 | function initialize_comm_if_necessary() { 35 | console.log("initializing comm") 36 | if (comm != null) { 37 | console.log("comm already initialized") 38 | return; 39 | } 40 | 41 | console.log("actually initializing comm") 42 | comm = Jupyter.notebook.kernel.comm_manager.new_comm('whyprofiler', {'initialized': true}) 43 | comm.on_msg(on_comm_message); 44 | console.log(log_prefix, "done initializing comm") 45 | } 46 | 47 | function get_css_class_for_severity(severity) { 48 | // TODO: convert float to number between 0 and 11 49 | return "whyprofiler-severity-" + severity; 50 | } 51 | 52 | function on_comm_message(msg) { 53 | console.log(log_prefix, "got comm message", msg.content.data); 54 | if (msg.content.data.msg_type == "analysis_ready") { 55 | console.log(log_prefix, "analysis ready"); 56 | update_ui_for_cell(cell, msg.content.data) 57 | } else if (msg.content.data.msg_type == "fix_ready") { 58 | console.log(log_prefix, "fix ready"); 59 | apply_fix(cell, msg.content.data); 60 | } 61 | 62 | console.log(log_prefix, "done handling comm message", msg.content.data); 63 | } 64 | 65 | function get_semgrep_issues_by_line(server_data) { 66 | return _.object(_.map(server_data.analysis.results, (res) => [res.end.line, [res.check_id, res.extra.message]])) 67 | } 68 | 69 | function intialize_and_cleanup_old_stuff(code_mirror) { 70 | code_mirror.setOption("gutters", [GUTTER]); 71 | _.each(_.range(code_mirror.lineCount()), (i)=>code_mirror.removeLineClass(i, "background")); 72 | _.each(_.range(code_mirror.lineCount()), (i)=>code_mirror.removeLineClass(i, "wrap")); 73 | _.each(line_widgets, (widget)=>widget.clear()); 74 | line_widgets = []; 75 | code_mirror.clearGutter(GUTTER); 76 | } 77 | 78 | function request_fix(check_id, code_mirror, rerun_after_fix, previous_time) { 79 | console.log(log_prefix, "should apply fix", check_id); 80 | comm.send({ 81 | msg_type : "apply_fix", 82 | check_id : check_id, 83 | code: code_mirror.getDoc().getValue(), 84 | rerun_after_fix : rerun_after_fix, 85 | previous_time : previous_time, 86 | }); 87 | } 88 | 89 | function apply_fix(cell, server_data) { 90 | console.log(log_prefix, "now applying fix", server_data); 91 | cell.code_mirror.getDoc().setValue(server_data.fixed_code); 92 | if (server_data.request.rerun_after_fix == true) { 93 | console.log("rerunning after fix"); 94 | cell.metadata.fixed = true; 95 | cell.metadata.previous_time = server_data.request.previous_time; 96 | Jupyter.notebook.execute_selected_cells(); 97 | } 98 | } 99 | 100 | function update_ui_for_cell(cell, server_data) { 101 | console.log(log_prefix, "starting to update ui for cell"); 102 | 103 | let code_mirror = cell.code_mirror; 104 | intialize_and_cleanup_old_stuff(code_mirror); 105 | 106 | var semgrep_issues_by_line = get_semgrep_issues_by_line(server_data); 107 | var total_time = _.values(server_data.line_timing).reduce((a,b) => a+b, 0); 108 | 109 | if (cell.metadata.fixed == true) { 110 | let element = document.createElement("div"); 111 | let previous_time = cell.metadata.previous_time; 112 | element.className = "whyprofiler-codemirror-successs" 113 | element.innerText = "Fix saved " + (previous_time - total_time).toFixed(1) + " seconds (runs " + Math.round((previous_time - total_time) / previous_time * 100) + "% faster)"; 114 | let widget = code_mirror.addLineWidget(code_mirror.lineCount()-1, element); 115 | line_widgets.push(widget); 116 | cell.metadata.fixed = false; 117 | cell.metadata.previous_time = 0; 118 | } 119 | 120 | for (const [lineno_str, timing] of Object.entries(server_data.line_timing)) { 121 | var lineno = parseInt(lineno_str); 122 | var relative_time = timing / total_time; 123 | 124 | if (timing < THRESHOLD_SECONDS || relative_time < THRESHOLD_PERCENTAGE) { 125 | continue; 126 | } 127 | 128 | // severity is between 0 and 10 129 | var severity = Math.round(relative_time * 10).toString(); 130 | console.log(log_prefix, "line", lineno, " has relative_time of", relative_time, "and severity", severity); 131 | code_mirror.addLineClass(lineno-1, "background", get_css_class_for_severity(severity)); 132 | 133 | var gutter_element = document.createElement("div"); 134 | gutter_element.innerText = timing.toFixed(1) + "s"; 135 | gutter_element.className = get_css_class_for_severity(severity); 136 | code_mirror.setGutterMarker(lineno-1, GUTTER, gutter_element); 137 | 138 | if (_.has(semgrep_issues_by_line, lineno)) { 139 | let [check_id, check_description] = semgrep_issues_by_line[lineno] 140 | console.log(log_prefix, "found semgrep issue for line", lineno, "issue:", check_id) 141 | gutter_element.textContent = gutter_element.textContent + " *" 142 | let semgrep_element = document.createElement("div"); 143 | semgrep_element.innerHTML = "
Recommendation:
" + check_description; 144 | 145 | let apply_button = document.createElement("button"); 146 | apply_button.innerText = "Apply Fix"; 147 | apply_button.className = "whyprofiler-codemirror-fix-apply-btn"; 148 | apply_button.addEventListener("click", () => request_fix(check_id, code_mirror, false, total_time)); 149 | semgrep_element.appendChild(apply_button) 150 | 151 | let rerun_button = document.createElement("button"); 152 | rerun_button.innerText = "Apply and Rerun"; 153 | rerun_button.className = "whyprofiler-codemirror-fix-apply-btn"; 154 | rerun_button.addEventListener("click", () => request_fix(check_id, code_mirror, true, total_time)); 155 | semgrep_element.appendChild(rerun_button) 156 | 157 | let widget = code_mirror.addLineWidget(lineno-1, semgrep_element, { "className" : "whyprofiler-codemirror-fix" }); 158 | code_mirror.addLineClass(lineno-1, "background", "whyprofiler-codemirror-fix-border"); 159 | line_widgets.push(widget); 160 | } 161 | } 162 | 163 | code_mirror.refresh(); 164 | console.log(log_prefix, "done updating ui for cell"); 165 | } 166 | 167 | function execute_codecell_callback (evt, data) { 168 | console.log(log_prefix, "execute callback") 169 | initialize_comm_if_necessary() 170 | // TODO: send data here so we know how to map it back to the right cm? 171 | //comm.send({"execution_request" : true}) 172 | cell = data.cell; 173 | } 174 | 175 | function add_css(url) { 176 | $('') 177 | .attr({ 178 | rel: 'stylesheet', 179 | href: requirejs.toUrl(url), 180 | type: 'text/css' 181 | }) 182 | .appendTo('head'); 183 | } 184 | 185 | function load_jupyter_extension () { 186 | window.onbeforeunload = () => undefined; 187 | add_css('./whyprofiler.css'); 188 | 189 | events.on('kernel_connected.Kernel', function() { 190 | console.log(log_prefix, "kernel connected") 191 | initialize_comm_if_necessary() 192 | }); 193 | 194 | Jupyter.notebook.config.loaded.then(function do_stuff_with_config () { 195 | console.log(log_prefix, "starting loading callbacks"); 196 | events.on('execute.CodeCell', execute_codecell_callback); 197 | console.log(log_prefix, "done loading callbacks"); 198 | }).catch(function on_error (reason) { 199 | console.error(log_prefix, 'Error:', reason); 200 | }); 201 | } 202 | 203 | return { 204 | load_jupyter_extension : load_jupyter_extension, 205 | load_ipython_extension : load_jupyter_extension 206 | }; 207 | }); -------------------------------------------------------------------------------- /whyprofiler/static/semgrep/use_orjson.yaml: -------------------------------------------------------------------------------- 1 | rules: 2 | - id: use_orjson 3 | pattern: | 4 | import json 5 | ... 6 | json.dumps($X) 7 | fix-regex: 8 | regex: 'import json' 9 | replacement: 'import orjson as json' 10 | metadata: 11 | fixable: true 12 | fix_installation: 'orjson' 13 | message: Use orjson for faster json dumps 14 | languages: [python] 15 | severity: ERROR 16 | -------------------------------------------------------------------------------- /whyprofiler/static/semgrep_experiments/README.txt: -------------------------------------------------------------------------------- 1 | experimental rules that shouldn't be run by default -------------------------------------------------------------------------------- /whyprofiler/static/semgrep_experiments/use_set_not_list.yaml: -------------------------------------------------------------------------------- 1 | rules: 2 | - id: use_set_not_list 3 | patterns: 4 | - pattern-not-inside: | 5 | ... 6 | ... = $X[...] 7 | ... 8 | - pattern-not-inside: | 9 | ... 10 | $X[...] = ... 11 | ... 12 | - pattern-not-inside: | 13 | ... 14 | ... = $X[...] 15 | ... 16 | - pattern-inside: | 17 | $X = list($INIT) 18 | ... 19 | for $Y in $Z: 20 | ... 21 | ... = <... ... in $X ...> 22 | - pattern: | 23 | for $Y in $Z: 24 | fix: | 25 | $X = set($INIT) 26 | for $Y in $Z: 27 | message: "use sets not lists" 28 | languages: [python] 29 | severity: ERROR 30 | 31 | 32 | # - pattern: | 33 | # $X = ... 34 | # ... 35 | # if <... ... in $X ...>: 36 | # ... 37 | 38 | 39 | #patterns: 40 | # - pattern-inside: | 41 | # $X = [$...ARGS] 42 | # ... 43 | # for $...LOOP: 44 | # ... 45 | # if $Y in $X: 46 | # ... 47 | # - pattern: $X = [$...ARGS] 48 | # fix: $X = {$...ARGS} -------------------------------------------------------------------------------- /whyprofiler/static/whyprofiler.css: -------------------------------------------------------------------------------- 1 | .whyprofiler-codemirror-gutter { 2 | width: 60px; 3 | } 4 | 5 | .whyprofiler-codemirror-fix { 6 | color: white; 7 | font-style: italic; 8 | font-size: 12px; 9 | } 10 | 11 | .whyprofiler-codemirror-successs { 12 | background-color: #90EE90; 13 | font-style: italic; 14 | font-size: 12px; 15 | } 16 | 17 | .whyprofiler-codemirror-fix-prefix { 18 | color: black; 19 | font-style: italic; 20 | font-size: 12px; 21 | display: inline; 22 | } 23 | 24 | .whyprofiler-codemirror-fix-border { 25 | border-width: 2px; 26 | border-style: solid; 27 | border-color: black; 28 | } 29 | 30 | .whyprofiler-codemirror-fix-apply-btn { 31 | color: black; 32 | font-style: italic; 33 | font-size: 10px; 34 | margin-left: 10px; 35 | margin-bottom: 5px; 36 | } 37 | 38 | /* 39 | light color palette: https://coolors.co/a71e34-b21e35-bd1f36-d24055-e66074-ed8392-f3a5b0-f7c0c8-f9ced4-fadbdf 40 | */ 41 | .whyprofiler-severity-10 { 42 | background-color: #a71e34ff; 43 | } 44 | 45 | .whyprofiler-severity-9 { 46 | background-color: #b21e35ff; 47 | } 48 | 49 | .whyprofiler-severity-8 { 50 | background-color: #bd1f36ff; 51 | } 52 | 53 | .whyprofiler-severity-7 { 54 | background-color: #d24055ff; 55 | } 56 | 57 | .whyprofiler-severity-6 { 58 | background-color: #e66074ff; 59 | } 60 | 61 | .whyprofiler-severity-5 { 62 | background-color: #ed8392ff; 63 | } 64 | 65 | .whyprofiler-severity-4 { 66 | background-color: #f3a5b0ff; 67 | } 68 | 69 | .whyprofiler-severity-3 { 70 | background-color: #f7c0c8ff; 71 | } 72 | 73 | .whyprofiler-severity-2 { 74 | background-color: #f9ced4ff; 75 | } 76 | 77 | .whyprofiler-severity-1 { 78 | background-color: #fadbdfff; 79 | } 80 | 81 | .whyprofiler-severity-0 { 82 | background-color: #ffffff; 83 | } 84 | 85 | /* darker color palette: https://coolors.co/641220-6e1423-85182a-a11d33-a71e34-b21e35-bd1f36-e66074-f3a5b0-fadbdf */ 86 | /* 87 | .whyprofiler-severity-10 { 88 | background-color: #641220ff; 89 | } 90 | 91 | .whyprofiler-severity-9 { 92 | background-color: #6e1423ff; 93 | } 94 | 95 | .whyprofiler-severity-8 { 96 | background-color: #85182aff; 97 | } 98 | 99 | .whyprofiler-severity-7 { 100 | background-color: #a11d33ff; 101 | } 102 | 103 | .whyprofiler-severity-6 { 104 | background-color: #a71e34ff; 105 | } 106 | 107 | .whyprofiler-severity-5 { 108 | background-color: #b21e35ff; 109 | } 110 | 111 | .whyprofiler-severity-4 { 112 | background-color: #bd1f36ff; 113 | } 114 | 115 | .whyprofiler-severity-3 { 116 | background-color: #e66074ff; 117 | } 118 | 119 | .whyprofiler-severity-2 { 120 | background-color: #f3a5b0ff; 121 | } 122 | 123 | .whyprofiler-severity-1 { 124 | background-color: #fadbdfff; 125 | } 126 | 127 | .whyprofiler-severity-0 { 128 | background-color: #ffffff; 129 | } 130 | */ --------------------------------------------------------------------------------