├── .gitignore ├── LICENSE.txt ├── README.md ├── pipreqsnb ├── __init__.py └── pipreqsnb.py ├── requirements.txt ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .ipynb 3 | *.pyc 4 | /dist/ 5 | /*.egg-info 6 | tests/ 7 | .ipynb_checkpoints/ 8 | test_params.txt -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Ivan Lengyel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pipreqsnb 2 | 3 | This is a very simple fully compatible pipreqs wrapper that supports python files and jupyter notebooks. 4 | 5 | - All the arguments and options are the same as pipreqs. 6 | - pipreqs commands are still valid 7 | 8 | __Please__ see [pipreqs](https://github.com/bndr/pipreqs/) documenation for more information. 9 | 10 | ## New 11 | - v 0.2.1: Single file support. You can either target a single python `file.py` or a single jupyter notebook 12 | `notebook.ipynb`, `=single_file_path`. 13 | - v 0.2.4: bugfix: bugfix for args with `-` 14 | 15 | ## Installation 16 | 17 | pip install pipreqsnb 18 | 19 | ## Usage 20 | 21 | 22 | Usage: 23 | pipreqsnb [options] 24 | 25 | Options: 26 | --use-local Use ONLY local package info instead of querying PyPI 27 | --pypi-server Use custom PyPi server 28 | --proxy Use Proxy, parameter will be passed to requests library. You can also just set the 29 | environments parameter in your terminal: 30 | $ export HTTP_PROXY="http://10.10.1.10:3128" 31 | $ export HTTPS_PROXY="https://10.10.1.10:1080" 32 | --debug Print debug information 33 | --ignore ... Ignore extra directories (sepparated by comma no space) 34 | --encoding Use encoding parameter for file open 35 | --savepath Save the list of requirements in the given file 36 | --print Output the list of requirements in the standard output 37 | --force Overwrite existing requirements.txt 38 | --diff Compare modules in requirements.txt to project imports. 39 | --clean Clean up requirements.txt by removing modules that are not imported in project. 40 | --no-pin Omit version of output packages. 41 | -------------------------------------------------------------------------------- /pipreqsnb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivanlen/pipreqsnb/928493fc28c9ec402c2dd32547a7da22318fcb61/pipreqsnb/__init__.py -------------------------------------------------------------------------------- /pipreqsnb/pipreqsnb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import ast 6 | import shutil 7 | import json 8 | import os 9 | 10 | pipreqs_options_store = ['use-local', 'debug', 'print', 'force', 'no-pin'] 11 | pipreqs_options_args = ['pypi-server', 'proxy', 'ignore', 'encoding', 'savepath', 'diff', 'clean'] 12 | 13 | 14 | def clean_invalid_lines_from_list_of_lines(list_of_lines): 15 | invalid_starts = ['!', '%'] 16 | valid_python_lines = [] 17 | for line in list_of_lines: 18 | if not any([line.startswith(x) for x in invalid_starts]): 19 | valid_python_lines.append(line) 20 | return valid_python_lines 21 | 22 | 23 | def get_import_string_from_source(source): 24 | imports = [] 25 | splitted = source.splitlines() 26 | tree = ast.parse(source) 27 | for node in ast.walk(tree): 28 | if any([isinstance(node, ast.Import), isinstance(node, ast.ImportFrom)]): 29 | imports.append(splitted[node.lineno - 1]) 30 | return imports 31 | 32 | 33 | def generate_pipreqs_str(args): 34 | pipreqs_str = '' 35 | for arg, val in args.__dict__.items(): 36 | if arg.replace('_','-') in pipreqs_options_store and val: 37 | pipreqs_str += ' --{}'.format(arg.replace('_','-')) 38 | elif arg.replace('_','-') in pipreqs_options_args and val is not None: 39 | pipreqs_str += ' --{} {}'.format(arg.replace('_','-'), val) 40 | pipreqs_str += ' {}'.format(args.path) 41 | return pipreqs_str 42 | 43 | 44 | def run_pipreqs(args): 45 | print('pipreqs {}'.format(args)) 46 | os.system('pipreqs {}'.format(args)) 47 | 48 | 49 | def get_ipynb_files(path, ignore_dirs=None): 50 | parsed_ignore = ['.ipynb_checkpoints'] 51 | 52 | if ignore_dirs: 53 | parsed_ignore_dirs = ignore_dirs.split(',') 54 | parsed_ignore.extend(parsed_ignore_dirs) 55 | 56 | ipynb_files = [] 57 | for root, dirs, files in os.walk(path): 58 | dirs[:] = [d for d in dirs if d not in parsed_ignore] 59 | for name in files: 60 | f_path = os.path.realpath(os.path.join(root, name)) 61 | ext = os.path.splitext(f_path)[1] 62 | if ext == '.ipynb': 63 | ipynb_files.append(f_path) 64 | return ipynb_files 65 | 66 | 67 | def path_is_file(path): 68 | if os.path.isdir(path): 69 | return False, None 70 | elif os.path.isfile(path): 71 | extension = os.path.splitext(path)[1] 72 | if extension == '.py': 73 | is_nb = False 74 | elif extension == '.ipynb': 75 | is_nb = True 76 | else: 77 | raise Exception('file {} has an invalid extension {}'.format(path, extension)) 78 | return True, is_nb 79 | else: 80 | raise Exception('{} if an invalid path'.format(path)) 81 | 82 | 83 | def set_requirements_savepath(args): 84 | if args.savepath is None: 85 | return os.path.join(os.path.dirname(args.path), 'requirements.txt') 86 | return args.savepath 87 | 88 | 89 | def main(): 90 | parser = argparse.ArgumentParser() 91 | for preqs_opt in pipreqs_options_store: 92 | parser.add_argument('--{}'.format(preqs_opt), action='store_true') 93 | for preqs_opt in pipreqs_options_args: 94 | parser.add_argument('--{}'.format(preqs_opt), type=str) 95 | parser.add_argument('path', nargs='?', default=os.getcwd()) 96 | args = parser.parse_args() 97 | input_path = args.path 98 | is_file, is_nb = path_is_file(input_path) 99 | 100 | temp_file_name = '_pipreqsnb_temp_file.py' 101 | temp_path_folder_name = '__temp_pipreqsnb_folder' 102 | ignore_dirs = args.ignore if 'ignore' in args else None 103 | if is_file: 104 | temp_saving_path = os.path.join(os.getcwd(), temp_path_folder_name) 105 | if is_nb: 106 | ipynb_files = [input_path] 107 | else: 108 | ipynb_files = [] 109 | os.makedirs(temp_saving_path, exist_ok=True) 110 | shutil.copyfile(input_path, os.path.join(temp_saving_path, temp_file_name)) 111 | else: 112 | ipynb_files = get_ipynb_files(input_path, ignore_dirs=ignore_dirs) 113 | temp_saving_path = os.path.join(input_path, temp_path_folder_name) 114 | temp_file = os.path.join(temp_saving_path, temp_file_name) 115 | imports = [] 116 | open_file_args = {} 117 | if args.encoding is not None: 118 | open_file_args['encoding'] = args.encoding 119 | for nb_file in ipynb_files: 120 | nb = json.load(open(nb_file, 'r', **open_file_args)) 121 | try: 122 | for n_cell, cell in enumerate(nb['cells']): 123 | if cell['cell_type'] == 'code': 124 | valid_lines = clean_invalid_lines_from_list_of_lines(cell['source']) 125 | source = ''.join(valid_lines) 126 | imports += get_import_string_from_source(source) 127 | except Exception as e: 128 | print( 129 | "Exception occurred while working on file {}, cell {}/{}".format(nb_file, n_cell + 1, len(nb['cells']))) 130 | raise e 131 | 132 | # hack to remove the indents if imports are inside functions 133 | imports = [i.lstrip() for i in imports] 134 | 135 | if is_file: 136 | args.savepath = set_requirements_savepath(args) 137 | args.path = temp_saving_path 138 | try: 139 | os.makedirs(temp_saving_path, exist_ok=True) 140 | with open(temp_file, 'a') as temp_file: 141 | for import_line in imports: 142 | temp_file.write('{}\n'.format(import_line)) 143 | pipreqs_args = generate_pipreqs_str(args) 144 | run_pipreqs(pipreqs_args) 145 | shutil.rmtree(temp_saving_path) 146 | except Exception as e: 147 | if os.path.isfile(temp_file): 148 | os.remove(temp_file) 149 | raise e 150 | 151 | 152 | if __name__ == '__main__': 153 | main() 154 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pipreqs==0.4.12 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open('README.md') as readme_file: 4 | readme = readme_file.read() 5 | 6 | setuptools.setup( 7 | name='pipreqsnb', 8 | version='0.2.4', 9 | description='Pipreqs with jupyter notebook support', 10 | url='https://github.com/ivanlen/pipreqsnb', 11 | author='Ivan Lengyel', 12 | author_email='ivalengy@gmail.com', 13 | entry_points={'console_scripts': ['pipreqsnb=pipreqsnb.pipreqsnb:main']}, 14 | long_description_content_type='text/markdown', 15 | long_description=readme, 16 | license='MIT License', 17 | license_files=("LICENSE.txt",), 18 | packages=setuptools.find_packages(), 19 | install_requires=['pipreqs'], 20 | zip_safe=False) 21 | --------------------------------------------------------------------------------