├── .gitignore ├── LICENSE ├── NotifAnalyzer.py ├── NotifAnalyzer_Autopsy.py ├── README.md ├── YPAFI.py ├── YPAReport.py ├── bring2lite ├── LICENSE ├── WAL_parser.py ├── __init__.py ├── gui.py ├── journal_parser.py ├── main.py ├── parser.py ├── potentially_parser.py ├── report_generator.py ├── scandir │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── PKG-INFO │ ├── README.rst │ ├── __init__.py │ ├── _scandir.c │ ├── benchmark.py │ ├── osdefs.h │ ├── scandir.egg-info │ │ ├── PKG-INFO │ │ ├── SOURCES.txt │ │ ├── dependency_links.txt │ │ └── top_level.txt │ ├── setup.cfg │ ├── setup.py │ ├── test │ │ ├── run_tests.py │ │ ├── test_scandir.py │ │ └── test_walk.py │ └── winreparse.h ├── six │ ├── @PaxHeader │ ├── LICENSE │ └── __init__.py ├── sqlite_parser.py ├── sqlparse │ ├── LICENSE │ ├── __init__.py │ ├── __main__.py │ ├── aligned_indent.py │ ├── cli.py │ ├── compat.py │ ├── engine │ │ └── grouping.py │ ├── exceptions.py │ ├── filter_stack.py │ ├── filterstokens.py │ ├── formatter.py │ ├── grouping.py │ ├── keywords.py │ ├── lexer.py │ ├── others.py │ ├── output.py │ ├── reindent.py │ ├── right_margin.py │ ├── sql.py │ ├── statement_splitter.py │ ├── tokens.py │ └── utils.py └── tqdm │ ├── LICENSE │ ├── __init__.py │ ├── __main__.py │ ├── _main.py │ ├── _monitor.py │ ├── _tqdm.py │ ├── _tqdm_gui.py │ ├── _tqdm_notebook.py │ ├── _tqdm_pandas.py │ ├── _utils.py │ ├── _version.py │ ├── asyncio.py │ ├── auto.py │ ├── autonotebook.py │ ├── cli.py │ ├── completion.sh │ ├── contrib │ ├── __init__.py │ ├── bells.py │ ├── concurrent.py │ ├── discord.py │ ├── itertools.py │ ├── telegram.py │ └── utils_worker.py │ ├── gui.py │ ├── keras.py │ ├── notebook.py │ ├── std.py │ ├── tests │ ├── py37_asyncio.py │ ├── tests_asyncio.py │ ├── tests_concurrent.py │ ├── tests_contrib.py │ ├── tests_itertools.py │ ├── tests_keras.py │ ├── tests_main.py │ ├── tests_notebook.py │ ├── tests_pandas.py │ ├── tests_perf.py │ ├── tests_synchronisation.py │ ├── tests_tqdm.py │ └── tests_version.py │ ├── tqdm.1 │ └── utils.py ├── bs4 ├── LICENSE ├── __init__.py ├── builder │ ├── __init__.py │ ├── _html5lib.py │ ├── _htmlparser.py │ └── _lxml.py ├── dammit.py ├── diagnose.py ├── element.py ├── testing.py └── tests │ ├── __init__.py │ ├── test_builder_registry.py │ ├── test_docs.py │ ├── test_html5lib.py │ ├── test_htmlparser.py │ ├── test_lxml.py │ ├── test_soup.py │ └── test_tree.py ├── crawler ├── __init__.py ├── tqdm │ ├── __init__.py │ ├── __main__.py │ ├── _main.py │ ├── _monitor.py │ ├── _tqdm.py │ ├── _tqdm_gui.py │ ├── _tqdm_notebook.py │ ├── _tqdm_pandas.py │ ├── _utils.py │ ├── _version.py │ ├── asyncio.py │ ├── auto.py │ ├── autonotebook.py │ ├── cli.py │ ├── completion.sh │ ├── contrib │ │ ├── __init__.py │ │ ├── bells.py │ │ ├── concurrent.py │ │ ├── discord.py │ │ ├── itertools.py │ │ ├── telegram.py │ │ └── utils_worker.py │ ├── gui.py │ ├── keras.py │ ├── notebook.py │ ├── std.py │ ├── tests │ │ ├── py37_asyncio.py │ │ ├── tests_asyncio.py │ │ ├── tests_concurrent.py │ │ ├── tests_contrib.py │ │ ├── tests_itertools.py │ │ ├── tests_keras.py │ │ ├── tests_main.py │ │ ├── tests_notebook.py │ │ ├── tests_pandas.py │ │ ├── tests_perf.py │ │ ├── tests_synchronisation.py │ │ ├── tests_tqdm.py │ │ └── tests_version.py │ ├── tqdm.1 │ └── utils.py └── wal_crawler.py ├── db ├── __init__.py └── db_functions.py ├── default.jpg ├── mdgMod ├── __init__.py └── mdg_modified.py ├── template_address_book.html ├── template_call_history.html ├── template_chats.html ├── template_phone_apps.html ├── template_photos.html ├── undark-LICENSE └── undark.exe /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.class 3 | */*.class 4 | .vscode/ -------------------------------------------------------------------------------- /NotifAnalyzer.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import time 4 | import sqlite3 5 | import json 6 | # It's only used for formatting, so we don't want to crash due to it. 7 | try: 8 | from lxml import etree 9 | except ImportError: 10 | pass 11 | import re 12 | 13 | PRAGMA_USER_VERSION = 'PRAGMA user_version' 14 | USER_VERSION = 'user_version' 15 | QUERY_ASSETS = 'SELECT RecordId, PrimaryId, nh.ParentId, AssetKey, AssetValue, WNSId, HandlerType, WNFEventName, \ 16 | SystemDataPropertySet, nh.CreatedTime, nh.ModifiedTime, n.Payload, n.Type, n.ArrivalTime, \ 17 | n.PayloadType, n.ExpiryTime, n.Id \ 18 | FROM NotificationHandler nh \ 19 | LEFT JOIN HandlerAssets ha ON ha.HandlerId = nh.RecordId \ 20 | LEFT JOIN Notification n ON nh.RecordId = n.HandlerId \ 21 | ORDER BY RecordId' 22 | ASSETS = "assets" 23 | 24 | def main(args): 25 | start_time = time.time() 26 | path = args.path 27 | jpath = args.json 28 | if not path: 29 | print("Path is required.") 30 | exit() 31 | if not jpath: 32 | print("JSON result path is required.") 33 | exit() 34 | data = process_db(path) 35 | print(str(data)) 36 | with open(jpath, 'w') as fp: 37 | json.dump(data, fp, indent=4) 38 | total_time = round(time.time() - start_time, 2) 39 | print('Elapsed time: ' + str(total_time) + 's') 40 | try: 41 | from win10toast import ToastNotifier 42 | ToastNotifier().show_toast('NotifAnalyzer', f'Finished processing notifications! Took {str(total_time)}s', duration = None) 43 | except Exception as e: 44 | pass 45 | 46 | def process_db(file): 47 | db_info = {} 48 | try: 49 | db_conn = sqlite3.connect(file) 50 | db_conn.row_factory = sqlite3.Row 51 | c = db_conn.cursor() 52 | c.execute(PRAGMA_USER_VERSION) 53 | db_info[USER_VERSION] = c.fetchone()[0] 54 | c.execute(QUERY_ASSETS) 55 | asset_data = [dict(row) for row in c.fetchall()] 56 | db_info[ASSETS] = process_assets(asset_data) 57 | except Exception as e: 58 | db_info = None 59 | print(str(e)) 60 | finally: 61 | c.close() 62 | db_conn.close() 63 | return db_info 64 | 65 | def process_assets(assets): 66 | processed_assets = {} 67 | for asset in assets: 68 | id = asset["RecordId"] 69 | if id in processed_assets: 70 | # Not new asset 71 | process_asset_key(asset, processed_assets[id]) 72 | process_notification(asset, dict_asset) 73 | else: 74 | # New asset 75 | dict_asset = {} 76 | dict_asset["HandlerId"] = id 77 | dict_asset["HandlerPrimaryId"] = asset["PrimaryId"] 78 | dict_asset["ParentId"] = asset["ParentId"] 79 | dict_asset["WNSId"] = asset["WNSId"] 80 | dict_asset["HandlerType"] = asset["HandlerType"] 81 | dict_asset["WNFEventName"] = asset["WNFEventName"] 82 | dict_asset["SystemDataPropertySet"] = asset["SystemDataPropertySet"] 83 | dict_asset["CreatedTime"] = asset["CreatedTime"] 84 | dict_asset["ModifiedTime"] = asset["ModifiedTime"] 85 | dict_asset["OtherAssets"] = [] 86 | dict_asset["Notifications"] = [] 87 | process_asset_key(asset, dict_asset) 88 | process_notification(asset, dict_asset) 89 | 90 | processed_assets[id] = dict_asset 91 | 92 | return processed_assets 93 | 94 | def process_asset_key(asset, dict_asset): 95 | if "AssetKey" not in asset: 96 | return 97 | asset_key = asset["AssetKey"] 98 | if asset_key == "DisplayName": 99 | dict_asset["AppName"] = asset["AssetValue"] 100 | elif asset_key: 101 | asset_pair = {asset_key: asset["AssetValue"]} 102 | if asset_pair not in dict_asset["OtherAssets"]: 103 | dict_asset["OtherAssets"].append(asset_pair) 104 | 105 | def process_notification(asset, dict_asset): 106 | if "Payload" not in asset: 107 | return 108 | payload = asset["Payload"] 109 | if payload: 110 | try: 111 | root = etree.fromstring(payload) 112 | payload = etree.tostring(root, pretty_print=True).decode() 113 | except Exception as e: 114 | print("Failed to format XML due to " + str(e) + ". Falling back newline after '>'") 115 | try: 116 | payload = re.sub(r'(>)', r'\1\n', payload.decode()) 117 | except Exception as e: 118 | print("Failed to format XML due to " + str(e) + ". Falling back to Payload as string") 119 | payload = str(payload) 120 | notif = { 121 | "Id": asset["Id"], 122 | "Payload": payload, 123 | "Type": asset["Type"], 124 | "ExpiryTime": asset["ExpiryTime"], 125 | "ArrivalTime": asset["ArrivalTime"], 126 | "PayloadType": asset["PayloadType"] 127 | } 128 | # Verify that there are no duplicate notifications 129 | if not any(x["Id"] == asset["Id"] for x in dict_asset["Notifications"]): 130 | dict_asset["Notifications"].append(notif) 131 | 132 | def setup_args(): 133 | parser = argparse.ArgumentParser() 134 | parser.add_argument('-p', '--path', type=str, help='Path to Notifications DB (wpndatabase.db)') 135 | parser.add_argument('-j', '--json', type=str, help='Path to result file in JSON') 136 | return parser.parse_args() 137 | 138 | if __name__ == "__main__": 139 | args = setup_args() 140 | main(args) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YourPhoneAnalyzer 2 | 3 | * [Installation](#windows-installation) 4 | * [Cite](#cite) 5 | * [Authors + Contacts](#authors) 6 | * [Test data](#test-data) 7 | * [Licenses](#licenses) 8 | 9 | Autopsy plugin made to extract information from the 'Your Phone' Windows 10 App 10 | 11 | # Windows installation 12 | 13 | ## Autopsy 14 | 15 | 1. Download as ZIP directly from here 16 | 2. Go to your Python Modules folder: 17 | 1. Open Autopsy 18 | 2. Tools > Python Plugins 19 | 3. Unzip the downloaded ZIP inside the folder opened by Autopsy 20 | 4. Restart or start Autopsy to compile all the libraries and files 21 | * If also using the Notifications plugin, you will need: 22 | 1. Python installed to run the external script (Python 3 recommended) 23 | 2. lxml for proper identation: `pip install lxml` (optional) 24 | 3. win10toast for a Notification at the end of processing (optional) 25 | 5. Open your case and run the YPA file ingest module 26 | 6. Run the YPA Report Module with the desired options 27 | 7. Open the report (HTML) 28 | 29 | ## Using Notification Analyzer as a Python script 30 | 31 | You only need Python (recommended Python 3) and lxml (`pip install lxml`). 32 | 33 | If you want a Windows Notification at the end of processing (optional): `pip install win10toast` 34 | 35 | You can run it as: 36 | 37 | `python NotifAnalyzer.py -p path_to_database/wpndatabase.db -j path_to_output.json` 38 | 39 | # Cite 40 | 41 | If you need to cite this work, please use the following reference: 42 | 43 | Domingues, Patricio, Miguel Frade, Luis Miguel Andrade, and João Victor Silva. "Digital forensic artifacts of the Your Phone application in Windows 10." *Digital Investigation* (2019). 44 | https://www.sciencedirect.com/science/article/pii/S1742287619301239 45 | 46 | # Authors 47 | 48 | YPA was developed by Luís Miguel Andrade, João Victor Silva, Patrício Domingues, and Miguel Frade. 49 | 50 | If you have any suggestion or find any bug, please contact us or create an issue in this repository. 51 | 52 | **Contacts:** 53 | 54 | Luís Andrade - luis.m.andrade@outlook.com 55 | 56 | João Silva - jvictor.reiss@gmail.com 57 | 58 | Patrício Domingues - patricio.domingues@ipleiria.pt 59 | 60 | Miguel Frade - miguel.frade@ipleiria.pt 61 | 62 | # Test data 63 | 64 | If you wish to test the module before a real case, you have some dummy databases here: https://www.dropbox.com/s/t2p4q3pxe8jyaot/YourPhone_test_DB_datasource.zip?dl=0 65 | 66 | # Licenses 67 | 68 | This module is licensed under GPL 3.0 69 | 70 | This module uses a modified version of mdegrazia's SQLite-Deleted-Records-Parser (https://github.com/mdegrazia/SQLite-Deleted-Records-Parser) which is licensed under GPL 3.0 71 | 72 | This module uses the binary form of Undark (https://github.com/inflex/undark). Undark's license can be located at the file undark-LICENSE 73 | -------------------------------------------------------------------------------- /bring2lite/LICENSE: -------------------------------------------------------------------------------- 1 | Original repository: https://github.com/bring2lite/bring2lite 2 | License: https://github.com/bring2lite/bring2lite#licence -------------------------------------------------------------------------------- /bring2lite/WAL_parser.py: -------------------------------------------------------------------------------- 1 | from .parser import Parser 2 | from .report_generator import ReportGenerator 3 | import logging 4 | import binascii 5 | import os 6 | from struct import * 7 | import hashlib 8 | 9 | 10 | d = {} 11 | class WALParser(Parser): 12 | MAGIC_NUMBERS = [ 13 | binascii.unhexlify(b'377f0682'), 14 | binascii.unhexlify(b'377f0683') 15 | ] 16 | FILE_FORMAT_VERSION = binascii.unhexlify(b'03007000') 17 | """defining the offsets which have to be added by processing each page""" 18 | WAL_HEADER_SIZE = 32 19 | FRAME_HEADER_SIZE = 24 20 | MAIN_DB_HEADER_SIZE = 100 21 | 22 | def __init__(self): 23 | self.logger = logging.getLogger("parser.WALParser") 24 | self.filename = "" 25 | self.rgen = ReportGenerator() 26 | 27 | def parse(self, filename, outname, format, sqlite_present=False): 28 | #tqdm.write(Fore.GREEN + "[+] " + "Start parsing main WAL file") 29 | self.filename = filename 30 | self.outname = outname 31 | # CSV = 0 | XML = 1 | JSON = 2 32 | self.forma = format 33 | if not os.path.isfile(self.filename): 34 | raise IOError 35 | self.filesize = os.stat(self.filename).st_size 36 | self._parse_header() 37 | self._parse_body(sqlite_present) 38 | return d 39 | 40 | def _parse_header(self): 41 | with open(self.filename, "rb") as f: 42 | self.logger.debug("read WAL header") 43 | header = f.read(32) 44 | 45 | self.wal_header = unpack('>IIIIIIII', header) 46 | self.logger.debug("extract header data: " + str(self.wal_header)) 47 | self.magic = self.wal_header[0] 48 | self.page_size = self.wal_header[2] 49 | self.checkpoint_sequence = self.wal_header[3] 50 | self.salt_1 = self.wal_header[4] 51 | 52 | self.logger.debug("end of WAL header parsing") 53 | 54 | def _parse_body(self, sqlite_present): 55 | self.framecount = int((self.filesize - self.WAL_HEADER_SIZE) / (self.FRAME_HEADER_SIZE + self.page_size)) 56 | main_hashes = [] 57 | global d 58 | d["wal"] = {} 59 | if sqlite_present: 60 | main_hashes = self._extract_sqlite_hashes() 61 | with open(self.filename, "rb") as f: 62 | self.logger.debug("read WAL body ") 63 | """loop over each page within the WAL""" 64 | for i in range(0, self.framecount): 65 | """calculating the offset to each frame with the defined statics""" 66 | frame_offset = i*(self.page_size + self.FRAME_HEADER_SIZE) + self.WAL_HEADER_SIZE 67 | f.seek(frame_offset) 68 | #TODO: maybe add the page number to the output 69 | frame_page_number = unpack('>I', f.read(4)) 70 | 71 | f.seek(frame_offset + self.FRAME_HEADER_SIZE) 72 | p = f.read(self.page_size) 73 | if hashlib.sha256(p).hexdigest() in main_hashes: 74 | continue 75 | 76 | result = super(WALParser, self)._parse_page(p, is_wal=True) 77 | if not isinstance(result, int): 78 | d["wal"][i] = result 79 | # self.logger.debug("WAL result: " + str(result)) 80 | # self.rgen.generateReport(os.path.abspath(self.outname + "/" + super(WALParser, self)._path_leaf(self.filename) + "/WALs/" + "/"), 81 | # str(i) + "-wal-frame", result) 82 | 83 | self.logger.debug("end of WAL body parsing") 84 | 85 | 86 | """ 87 | Experiment to look for active and inactive pages in the main file to reduce parsing time 88 | """ 89 | def _extract_sqlite_hashes(self): 90 | main_page_hashes = [] 91 | with open(self.filename[0:-4], "rb") as f: 92 | f.seek(16) 93 | header = f.read(self.MAIN_DB_HEADER_SIZE - 44) 94 | sqlite_header = unpack('>HBBBBBBIIIIIIIIIIII', header) 95 | page_size = sqlite_header[0] 96 | if page_size == 1: 97 | page_size = 65536 98 | file_size = os.stat(self.filename[0:-4]).st_size 99 | max = file_size / page_size 100 | for counter in range(1, int(max)): 101 | f.seek(counter * page_size) 102 | one_page = f.read(page_size) 103 | main_page_hashes.append(hashlib.sha256(one_page).hexdigest()) 104 | 105 | return main_page_hashes 106 | -------------------------------------------------------------------------------- /bring2lite/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labcif/YPA/b984ab0719e554926e1f46bd9f669ca3c6cc15d4/bring2lite/__init__.py -------------------------------------------------------------------------------- /bring2lite/journal_parser.py: -------------------------------------------------------------------------------- 1 | from .parser import Parser 2 | from .report_generator import ReportGenerator 3 | import logging 4 | import binascii 5 | import os 6 | import re 7 | from struct import * 8 | 9 | 10 | class JournalParser(Parser): 11 | MAGIC_NUMBER = binascii.unhexlify(b'd9d505f920a163d7') 12 | 13 | def __init__(self): 14 | self.logger = logging.getLogger("parser.JournalParser") 15 | self.rgen = ReportGenerator() 16 | 17 | def parse(self, filename, outname, format, page_size): 18 | self.filename = filename 19 | self.outname = outname 20 | # CSV = 0 | XML = 1 | JSON = 2 21 | self.format = format 22 | if not os.path.isfile(self.filename): 23 | raise IOError 24 | self.filesize = os.stat(self.filename).st_size 25 | self.page_size = page_size 26 | self._parse_header() 27 | self._parse_body() 28 | 29 | def _parse_header(self): 30 | self.logger.debug("read journal header ") 31 | with open(self.filename, 'rb') as f: 32 | """searching the disk sector sized offset to find content""" 33 | str = f.read(512) 34 | counter = 2 35 | to_test = unpack('>H', str[counter: counter + 2])[0] 36 | 37 | while to_test == 0: 38 | counter += 2 39 | try: 40 | to_test = unpack('>H', str[counter: counter + 2])[0] 41 | except error: 42 | to_test = None 43 | #counter -= 2 44 | ''' 45 | while to_test is None: 46 | 47 | if to_test: 48 | break 49 | counter += 1 50 | ''' 51 | if counter >= 512 and counter < 1024: 52 | counter = 512 53 | 54 | self.header_padding = counter 55 | 56 | if self.header_padding == 0: 57 | with open(self.filename, "rb") as f: 58 | self.logger.debug("read WAL header") 59 | header = f.read(28) 60 | 61 | self.journal_header = unpack('>QIIIII', header) 62 | self.magic = self.journal_header[0] 63 | self.page_count = self.journal_header[1] 64 | self.initial_size_of_the_database_in_pages = self.journal_header[3] 65 | if self.journal_header[4] is not 0: 66 | self.header_padding = self.journal_header[4] 67 | if self.page_size is 0: 68 | self.page_size = self.journal_header[5] 69 | self.logger.debug("end read journal header ") 70 | if not self.page_size: 71 | raise ValueError 72 | 73 | def _parse_body(self): 74 | checksum_size = pagenumber_size = 4 75 | 76 | self.page_counter = int((self.filesize - self.header_padding) / (checksum_size + pagenumber_size + self.page_size)) 77 | with open(self.filename, "rb") as f: 78 | self.logger.debug("read journal body ") 79 | for i in range(0, self.page_counter): 80 | page_offset = i*(self.page_size + pagenumber_size) + self.header_padding 81 | if(i > 0): 82 | page_offset += i * checksum_size 83 | 84 | f.seek(page_offset) 85 | page_number = unpack('>I', f.read(4)) 86 | 87 | f.seek(page_offset + pagenumber_size) 88 | page = f.read(self.page_size) 89 | 90 | result = super(JournalParser, self)._parse_page(page) 91 | if not isinstance(result, int): 92 | self.rgen.generateReport(os.path.abspath(self.outname + "/" + super(JournalParser, self)._path_leaf(self.filename) + "/Journals/" + "/"), str(i) + 93 | "-jornal-page", result, self.format, schema=["No schema found"]) 94 | self.logger.debug("end body pardsing") 95 | 96 | def match_zeros(strg, search=re.compile(r'[^0]').search): 97 | return not bool(search(strg)) -------------------------------------------------------------------------------- /bring2lite/main.py: -------------------------------------------------------------------------------- 1 | from gui import GUI 2 | import logging 3 | import os 4 | 5 | 6 | def main(temp_dir): 7 | logger = logging.getLogger('parser') 8 | logger.setLevel(logging.DEBUG) 9 | debug_log_path = os.path.join(temp_dir, 'debug.log') 10 | fh = logging.FileHandler(debug_log_path) 11 | fh.setLevel(logging.DEBUG) 12 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 13 | fh.setFormatter(formatter) 14 | logger.addHandler(fh) 15 | 16 | return GUI() 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /bring2lite/report_generator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import hashlib 3 | from tqdm import tqdm 4 | import binascii 5 | 6 | class ReportGenerator: 7 | def __init__(self): 8 | self.my_path = "" 9 | 10 | def generateReport(self, path, filename, data, format="CSV", schema=["No schema found"]): 11 | if data is None: 12 | return 13 | 14 | if not os.path.exists(path): 15 | os.makedirs(path) 16 | 17 | # if format: 18 | # with open(path+filename+'.csv', 'w', newline='') as f: 19 | # writer = csv.writer(f) 20 | # writer.writerows(data) 21 | if data: 22 | out = "" 23 | for datatype in schema: 24 | out += str(datatype) + "," 25 | out += "\n" 26 | 27 | for frame in data: 28 | if isinstance(frame, list): 29 | for y in frame: 30 | if self.is_text(y[0]): 31 | try: 32 | out += str(y[1].decode('utf-8')) + "," 33 | except UnicodeDecodeError: 34 | out +=str(y[1]) + "," 35 | continue 36 | else: 37 | out += str(y[1]) + "," 38 | out += "\n" 39 | out += "++++++++++++++++++++++++++++\n" 40 | try: 41 | with open(path + "/" + filename + '.log', "a") as f: 42 | f.write(out) 43 | except UnicodeEncodeError: 44 | tqdm.write("can not write the record because of unicode errors") 45 | 46 | self.print_hash(path + "/" + filename + '.log') 47 | 48 | def generate_schema_report(self, path, filename, data, csv): 49 | if data is None: 50 | return 51 | 52 | if not os.path.exists(path): 53 | os.makedirs(path) 54 | 55 | out = "" 56 | with open(path + "/" + filename + '.log', "a") as f: 57 | for key, value in data.items(): 58 | # out += str(key) + ", " 59 | if isinstance(value, list): 60 | for y in value: 61 | out += str(y) + ", " 62 | out += "\n" 63 | out += "++++++++++++++++++++++++++++\n" 64 | 65 | f.write(out) 66 | 67 | self.print_hash(path + "/" + filename + '.log') 68 | 69 | def generate_freeblock_report(self, path, filename, freeblocks): 70 | if freeblocks is None: 71 | return 72 | 73 | if not os.path.exists(path): 74 | os.makedirs(path) 75 | 76 | with open(path + "/" + filename + '.log', "a") as f: 77 | for solutions in freeblocks: 78 | for s in solutions: 79 | if isinstance(s[0], list): 80 | f.write(str(s) + ",") 81 | else: 82 | if self.is_text(s[0]): 83 | f.write(str(s[1].decode('utf-8')) + ",") 84 | else: 85 | f.write(str(s[1]) + ",") 86 | f.write("\n" + "###################" + "\n") 87 | 88 | self.print_hash(path + "/" + filename + '.log') 89 | 90 | def is_text(self, tester): 91 | return tester == 'TEXT' 92 | 93 | def print_hash(self, filename): 94 | with open(filename, "rb") as f: 95 | d = f.read() 96 | tqdm.write("sha-256: " + filename + '\t => \t' + str(hashlib.sha256(d).hexdigest())) -------------------------------------------------------------------------------- /bring2lite/scandir/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Ben Hoyt 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of Ben Hoyt nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without specific 16 | prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /bring2lite/scandir/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.py 2 | include *.c 3 | include *.h 4 | include *.txt 5 | include *.rst 6 | include test/*.py 7 | -------------------------------------------------------------------------------- /bring2lite/scandir/osdefs.h: -------------------------------------------------------------------------------- 1 | // from CPython 2 | #ifndef Py_OSDEFS_H 3 | #define Py_OSDEFS_H 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | 9 | /* Operating system dependencies */ 10 | 11 | #ifdef MS_WINDOWS 12 | #define SEP L'\\' 13 | #define ALTSEP L'/' 14 | #define MAXPATHLEN 256 15 | #define DELIM L';' 16 | #endif 17 | 18 | /* Filename separator */ 19 | #ifndef SEP 20 | #define SEP L'/' 21 | #endif 22 | 23 | /* Max pathname length */ 24 | #ifdef __hpux 25 | #include 26 | #include 27 | #ifndef PATH_MAX 28 | #define PATH_MAX MAXPATHLEN 29 | #endif 30 | #endif 31 | 32 | #ifndef MAXPATHLEN 33 | #if defined(PATH_MAX) && PATH_MAX > 1024 34 | #define MAXPATHLEN PATH_MAX 35 | #else 36 | #define MAXPATHLEN 1024 37 | #endif 38 | #endif 39 | 40 | /* Search path entry delimiter */ 41 | #ifndef DELIM 42 | #define DELIM L':' 43 | #endif 44 | 45 | #ifdef __cplusplus 46 | } 47 | #endif 48 | #endif /* !Py_OSDEFS_H */ 49 | -------------------------------------------------------------------------------- /bring2lite/scandir/scandir.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE.txt 2 | MANIFEST.in 3 | README.rst 4 | _scandir.c 5 | benchmark.py 6 | osdefs.h 7 | scandir.py 8 | setup.py 9 | winreparse.h 10 | scandir.egg-info/PKG-INFO 11 | scandir.egg-info/SOURCES.txt 12 | scandir.egg-info/dependency_links.txt 13 | scandir.egg-info/top_level.txt 14 | test/run_tests.py 15 | test/test_scandir.py 16 | test/test_walk.py -------------------------------------------------------------------------------- /bring2lite/scandir/scandir.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /bring2lite/scandir/scandir.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | _scandir 2 | scandir 3 | -------------------------------------------------------------------------------- /bring2lite/scandir/setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = 3 | tag_date = 0 4 | 5 | -------------------------------------------------------------------------------- /bring2lite/scandir/setup.py: -------------------------------------------------------------------------------- 1 | """Run "python setup.py install" to install scandir.""" 2 | 3 | try: 4 | from setuptools import setup, Extension 5 | from setuptools.command.build_ext import build_ext as base_build_ext 6 | except ImportError: 7 | import warnings 8 | import sys 9 | val = sys.exc_info()[1] 10 | 11 | warnings.warn("import of setuptools failed %r" % val) 12 | from distutils.core import setup, Extension 13 | from distutils.command.build_ext import build_ext as base_build_ext 14 | 15 | import os 16 | import re 17 | import sys 18 | import logging 19 | 20 | # Get version without importing scandir because that will lock the 21 | # .pyd file (if scandir is already installed) so it can't be 22 | # overwritten during the install process 23 | with open(os.path.join(os.path.dirname(__file__), 'scandir.py')) as f: 24 | for line in f: 25 | match = re.match(r"__version__.*'([0-9.]+)'", line) 26 | if match: 27 | version = match.group(1) 28 | break 29 | else: 30 | raise Exception("Couldn't find version in setup.py") 31 | 32 | with open('README.rst') as f: 33 | long_description = f.read() 34 | 35 | 36 | class BuildExt(base_build_ext): 37 | 38 | # the extension is optional since in case of lack of c the api 39 | # there is a ctypes fallback and a slow python fallback 40 | 41 | def build_extension(self, ext): 42 | try: 43 | base_build_ext.build_extension(self, ext) 44 | except Exception: 45 | exception = sys.exc_info()[0] 46 | logging.warn("building the %s failed with %s", ext.name, exception) 47 | 48 | extension = Extension('_scandir', ['_scandir.c'], optional=True) 49 | 50 | 51 | setup( 52 | name='scandir', 53 | version=version, 54 | author='Ben Hoyt', 55 | author_email='benhoyt@gmail.com', 56 | url='https://github.com/benhoyt/scandir', 57 | license='New BSD License', 58 | description='scandir, a better directory iterator and faster os.walk()', 59 | long_description=long_description, 60 | py_modules=['scandir'], 61 | ext_modules=[extension], 62 | classifiers=[ 63 | 'Development Status :: 5 - Production/Stable', 64 | 'Intended Audience :: Developers', 65 | 'Operating System :: OS Independent', 66 | 'License :: OSI Approved :: BSD License', 67 | 'Programming Language :: Python', 68 | 'Topic :: System :: Filesystems', 69 | 'Topic :: System :: Operating System', 70 | 'Programming Language :: Python', 71 | 'Programming Language :: Python :: 2', 72 | 'Programming Language :: Python :: 2.7', 73 | 'Programming Language :: Python :: 3', 74 | 'Programming Language :: Python :: 3.4', 75 | 'Programming Language :: Python :: 3.5', 76 | 'Programming Language :: Python :: 3.6', 77 | 'Programming Language :: Python :: 3.7', 78 | 'Programming Language :: Python :: Implementation :: CPython', 79 | ], cmdclass={'build_ext': BuildExt}, 80 | ) 81 | -------------------------------------------------------------------------------- /bring2lite/scandir/test/run_tests.py: -------------------------------------------------------------------------------- 1 | """Run all unit tests.""" 2 | 3 | import glob 4 | import os 5 | import sys 6 | import unittest 7 | 8 | 9 | def main(): 10 | test_dir = os.path.dirname(os.path.abspath(__file__)) 11 | test_files = glob.glob(os.path.join(test_dir, 'test_*.py')) 12 | test_names = [os.path.basename(f)[:-3] for f in test_files] 13 | 14 | sys.path.insert(0, os.path.join(test_dir, '..')) 15 | 16 | suite = unittest.defaultTestLoader.loadTestsFromNames(test_names) 17 | result = unittest.TextTestRunner(verbosity=2).run(suite) 18 | sys.exit(1 if (result.errors or result.failures) else 0) 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /bring2lite/scandir/winreparse.h: -------------------------------------------------------------------------------- 1 | #ifndef Py_WINREPARSE_H 2 | #define Py_WINREPARSE_H 3 | 4 | #ifdef MS_WINDOWS 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /* The following structure was copied from 12 | http://msdn.microsoft.com/en-us/library/ff552012.aspx as the required 13 | include doesn't seem to be present in the Windows SDK (at least as included 14 | with Visual Studio Express). */ 15 | typedef struct _REPARSE_DATA_BUFFER { 16 | ULONG ReparseTag; 17 | USHORT ReparseDataLength; 18 | USHORT Reserved; 19 | union { 20 | struct { 21 | USHORT SubstituteNameOffset; 22 | USHORT SubstituteNameLength; 23 | USHORT PrintNameOffset; 24 | USHORT PrintNameLength; 25 | ULONG Flags; 26 | WCHAR PathBuffer[1]; 27 | } SymbolicLinkReparseBuffer; 28 | 29 | struct { 30 | USHORT SubstituteNameOffset; 31 | USHORT SubstituteNameLength; 32 | USHORT PrintNameOffset; 33 | USHORT PrintNameLength; 34 | WCHAR PathBuffer[1]; 35 | } MountPointReparseBuffer; 36 | 37 | struct { 38 | UCHAR DataBuffer[1]; 39 | } GenericReparseBuffer; 40 | }; 41 | } REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; 42 | 43 | #define REPARSE_DATA_BUFFER_HEADER_SIZE FIELD_OFFSET(REPARSE_DATA_BUFFER,\ 44 | GenericReparseBuffer) 45 | #define MAXIMUM_REPARSE_DATA_BUFFER_SIZE ( 16 * 1024 ) 46 | 47 | #ifdef __cplusplus 48 | } 49 | #endif 50 | 51 | #endif /* MS_WINDOWS */ 52 | 53 | #endif /* !Py_WINREPARSE_H */ 54 | -------------------------------------------------------------------------------- /bring2lite/six/@PaxHeader: -------------------------------------------------------------------------------- 1 | 22 mtime=1590074733.0 2 | -------------------------------------------------------------------------------- /bring2lite/six/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2020 Benjamin Peterson 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Andi Albrecht 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of the authors nor the names of its contributors may be 13 | used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /bring2lite/sqlparse/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | """Parse SQL statements.""" 10 | 11 | # Setup namespace 12 | import sql 13 | import cli 14 | import tokens 15 | import formatter 16 | 17 | from filter_stack import FilterStack 18 | from others import SerializerUnicode 19 | from compat import text_type 20 | 21 | __version__ = '0.3.1' 22 | __all__ = ['formatter', 'sql', 'tokens', 'cli'] 23 | 24 | 25 | def parse(sql, encoding=None): 26 | """Parse sql and return a list of statements. 27 | 28 | :param sql: A string containing one or more SQL statements. 29 | :param encoding: The encoding of the statement (optional). 30 | :returns: A tuple of :class:`~sqlparse.sql.Statement` instances. 31 | """ 32 | return tuple(parsestream(sql, encoding)) 33 | 34 | 35 | def parsestream(stream, encoding=None): 36 | """Parses sql statements from file-like object. 37 | 38 | :param stream: A file-like object. 39 | :param encoding: The encoding of the stream contents (optional). 40 | :returns: A generator of :class:`~sqlparse.sql.Statement` instances. 41 | """ 42 | stack = FilterStack() 43 | stack.enable_grouping() 44 | return stack.run(stream, encoding) 45 | 46 | 47 | def format(sql, encoding=None, **options): 48 | """Format *sql* according to *options*. 49 | 50 | Available options are documented in :ref:`formatting`. 51 | 52 | In addition to the formatting options this function accepts the 53 | keyword "encoding" which determines the encoding of the statement. 54 | 55 | :returns: The formatted SQL statement as string. 56 | """ 57 | stack = FilterStack() 58 | options = formatter.validate_options(options) 59 | stack = formatter.build_filter_stack(stack, options) 60 | stack.postprocess.append(SerializerUnicode()) 61 | return u''.join(stack.run(sql, encoding)) 62 | 63 | 64 | def split(sql, encoding=None): 65 | """Split *sql* into single statements. 66 | 67 | :param sql: A string containing one or more SQL statements. 68 | :param encoding: The encoding of the statement (optional). 69 | :returns: A list of strings. 70 | """ 71 | stack = FilterStack() 72 | return [text_type(stmt).strip() for stmt in stack.run(sql, encoding)] 73 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 5 | # 6 | # 7 | # This module is part of python-sqlparse and is released under 8 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 9 | 10 | """Entrypoint module for `python -m sqlparse`. 11 | 12 | Why does this file exist, and why __main__? For more info, read: 13 | - https://www.python.org/dev/peps/pep-0338/ 14 | - https://docs.python.org/2/using/cmdline.html#cmdoption-m 15 | - https://docs.python.org/3/using/cmdline.html#cmdoption-m 16 | """ 17 | 18 | import sys 19 | 20 | from cli import main 21 | 22 | if __name__ == '__main__': 23 | sys.exit(main()) 24 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | """Python 2/3 compatibility. 10 | 11 | This module only exists to avoid a dependency on six 12 | for very trivial stuff. We only need to take care of 13 | string types, buffers and metaclasses. 14 | 15 | Parts of the code is copied directly from six: 16 | https://bitbucket.org/gutworth/six 17 | """ 18 | 19 | import sys 20 | from io import TextIOBase 21 | 22 | PY2 = sys.version_info[0] == 2 23 | PY3 = sys.version_info[0] == 3 24 | 25 | 26 | if PY3: 27 | def unicode_compatible(cls): 28 | return cls 29 | 30 | text_type = str 31 | string_types = (str,) 32 | from io import StringIO 33 | file_types = (StringIO, TextIOBase) 34 | 35 | 36 | elif PY2: 37 | def unicode_compatible(cls): 38 | cls.__unicode__ = cls.__str__ 39 | cls.__str__ = lambda x: x.__unicode__().encode('utf-8') 40 | return cls 41 | 42 | text_type = unicode 43 | string_types = (str, unicode,) 44 | from StringIO import StringIO 45 | file_types = (file, StringIO, TextIOBase) 46 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | """Exceptions used in this package.""" 10 | 11 | 12 | class SQLParseError(Exception): 13 | """Base class for exceptions in this module.""" 14 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/filter_stack.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | """filter""" 10 | 11 | import lexer 12 | import grouping 13 | from statement_splitter import StatementSplitter 14 | 15 | 16 | class FilterStack(object): 17 | def __init__(self): 18 | self.preprocess = [] 19 | self.stmtprocess = [] 20 | self.postprocess = [] 21 | self._grouping = False 22 | 23 | def enable_grouping(self): 24 | self._grouping = True 25 | 26 | def run(self, sql, encoding=None): 27 | stream = lexer.tokenize(sql, encoding) 28 | # Process token stream 29 | for filter_ in self.preprocess: 30 | stream = filter_.process(stream) 31 | 32 | stream = StatementSplitter().process(stream) 33 | 34 | # Output: Stream processed Statements 35 | for stmt in stream: 36 | if self._grouping: 37 | stmt = grouping.group(stmt) 38 | 39 | for filter_ in self.stmtprocess: 40 | filter_.process(stmt) 41 | 42 | for filter_ in self.postprocess: 43 | stmt = filter_.process(stmt) 44 | 45 | yield stmt 46 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/filterstokens.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import tokens as T 10 | from compat import text_type 11 | 12 | 13 | class _CaseFilter(object): 14 | ttype = None 15 | 16 | def __init__(self, case=None): 17 | case = case or 'upper' 18 | self.convert = getattr(text_type, case) 19 | 20 | def process(self, stream): 21 | for ttype, value in stream: 22 | if ttype in self.ttype: 23 | value = self.convert(value) 24 | yield ttype, value 25 | 26 | 27 | class KeywordCaseFilter(_CaseFilter): 28 | ttype = T.Keyword 29 | 30 | 31 | class IdentifierCaseFilter(_CaseFilter): 32 | ttype = T.Name, T.String.Symbol 33 | 34 | def process(self, stream): 35 | for ttype, value in stream: 36 | if ttype in self.ttype and value.strip()[0] != '"': 37 | value = self.convert(value) 38 | yield ttype, value 39 | 40 | 41 | class TruncateStringFilter(object): 42 | def __init__(self, width, char): 43 | self.width = width 44 | self.char = char 45 | 46 | def process(self, stream): 47 | for ttype, value in stream: 48 | if ttype != T.Literal.String.Single: 49 | yield ttype, value 50 | continue 51 | 52 | if value[:2] == "''": 53 | inner = value[2:-2] 54 | quote = "''" 55 | else: 56 | inner = value[1:-1] 57 | quote = "'" 58 | 59 | if len(inner) > self.width: 60 | value = ''.join((quote, inner[:self.width], self.char, quote)) 61 | yield ttype, value 62 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/lexer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | """SQL Lexer""" 10 | 11 | # This code is based on the SqlLexer in pygments. 12 | # http://pygments.org/ 13 | # It's separated from the rest of pygments to increase performance 14 | # and to allow some customizations. 15 | 16 | import tokens 17 | from keywords import SQL_REGEX 18 | from compat import text_type, file_types 19 | from utils import consume 20 | 21 | 22 | class Lexer(object): 23 | """Lexer 24 | Empty class. Leaving for backwards-compatibility 25 | """ 26 | 27 | @staticmethod 28 | def get_tokens(text, encoding=None): 29 | """ 30 | Return an iterable of (tokentype, value) pairs generated from 31 | `text`. If `unfiltered` is set to `True`, the filtering mechanism 32 | is bypassed even if filters are defined. 33 | 34 | Also preprocess the text, i.e. expand tabs and strip it if 35 | wanted and applies registered filters. 36 | 37 | Split ``text`` into (tokentype, text) pairs. 38 | 39 | ``stack`` is the initial stack (default: ``['root']``) 40 | """ 41 | if isinstance(text, file_types): 42 | text = text.read() 43 | 44 | if isinstance(text, text_type): 45 | pass 46 | elif isinstance(text, bytes): 47 | if encoding: 48 | text = text.decode(encoding) 49 | else: 50 | try: 51 | text = text.decode('utf-8') 52 | except UnicodeDecodeError: 53 | text = text.decode('unicode-escape') 54 | else: 55 | raise TypeError(u"Expected text or file-like object, got {!r}". 56 | format(type(text))) 57 | 58 | iterable = enumerate(text) 59 | for pos, char in iterable: 60 | for rexmatch, action in SQL_REGEX: 61 | m = rexmatch(text, pos) 62 | 63 | if not m: 64 | continue 65 | elif isinstance(action, tokens._TokenType): 66 | yield action, m.group() 67 | elif callable(action): 68 | yield action(m.group()) 69 | 70 | consume(iterable, m.end() - pos - 1) 71 | break 72 | else: 73 | yield tokens.Error, char 74 | 75 | 76 | def tokenize(sql, encoding=None): 77 | """Tokenize sql. 78 | 79 | Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream 80 | of ``(token type, value)`` items. 81 | """ 82 | return Lexer().get_tokens(sql, encoding) 83 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/others.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import sql, tokens as T 10 | from utils import split_unquoted_newlines 11 | 12 | 13 | class StripCommentsFilter(object): 14 | @staticmethod 15 | def _process(tlist): 16 | def get_next_comment(): 17 | # TODO(andi) Comment types should be unified, see related issue38 18 | return tlist.token_next_by(i=sql.Comment, t=T.Comment) 19 | 20 | tidx, token = get_next_comment() 21 | while token: 22 | pidx, prev_ = tlist.token_prev(tidx, skip_ws=False) 23 | nidx, next_ = tlist.token_next(tidx, skip_ws=False) 24 | # Replace by whitespace if prev and next exist and if they're not 25 | # whitespaces. This doesn't apply if prev or next is a parenthesis. 26 | if (prev_ is None or next_ is None 27 | or prev_.is_whitespace or prev_.match(T.Punctuation, '(') 28 | or next_.is_whitespace or next_.match(T.Punctuation, ')')): 29 | # Insert a whitespace to ensure the following SQL produces 30 | # a valid SQL (see #425). For example: 31 | # 32 | # Before: select a--comment\nfrom foo 33 | # After: select a from foo 34 | if prev_ is not None and next_ is None: 35 | tlist.tokens.insert(tidx, sql.Token(T.Whitespace, ' ')) 36 | tlist.tokens.remove(token) 37 | else: 38 | tlist.tokens[tidx] = sql.Token(T.Whitespace, ' ') 39 | 40 | tidx, token = get_next_comment() 41 | 42 | def process(self, stmt): 43 | [self.process(sgroup) for sgroup in stmt.get_sublists()] 44 | StripCommentsFilter._process(stmt) 45 | return stmt 46 | 47 | 48 | class StripWhitespaceFilter(object): 49 | def _stripws(self, tlist): 50 | func_name = '_stripws_{cls}'.format(cls=type(tlist).__name__) 51 | func = getattr(self, func_name.lower(), self._stripws_default) 52 | func(tlist) 53 | 54 | @staticmethod 55 | def _stripws_default(tlist): 56 | last_was_ws = False 57 | is_first_char = True 58 | for token in tlist.tokens: 59 | if token.is_whitespace: 60 | token.value = '' if last_was_ws or is_first_char else ' ' 61 | last_was_ws = token.is_whitespace 62 | is_first_char = False 63 | 64 | def _stripws_identifierlist(self, tlist): 65 | # Removes newlines before commas, see issue140 66 | last_nl = None 67 | for token in list(tlist.tokens): 68 | if last_nl and token.ttype is T.Punctuation and token.value == ',': 69 | tlist.tokens.remove(last_nl) 70 | last_nl = token if token.is_whitespace else None 71 | 72 | # next_ = tlist.token_next(token, skip_ws=False) 73 | # if (next_ and not next_.is_whitespace and 74 | # token.ttype is T.Punctuation and token.value == ','): 75 | # tlist.insert_after(token, sql.Token(T.Whitespace, ' ')) 76 | return self._stripws_default(tlist) 77 | 78 | def _stripws_parenthesis(self, tlist): 79 | while tlist.tokens[1].is_whitespace: 80 | tlist.tokens.pop(1) 81 | while tlist.tokens[-2].is_whitespace: 82 | tlist.tokens.pop(-2) 83 | self._stripws_default(tlist) 84 | 85 | def process(self, stmt, depth=0): 86 | [self.process(sgroup, depth + 1) for sgroup in stmt.get_sublists()] 87 | self._stripws(stmt) 88 | if depth == 0 and stmt.tokens and stmt.tokens[-1].is_whitespace: 89 | stmt.tokens.pop(-1) 90 | return stmt 91 | 92 | 93 | class SpacesAroundOperatorsFilter(object): 94 | @staticmethod 95 | def _process(tlist): 96 | 97 | ttypes = (T.Operator, T.Comparison) 98 | tidx, token = tlist.token_next_by(t=ttypes) 99 | while token: 100 | nidx, next_ = tlist.token_next(tidx, skip_ws=False) 101 | if next_ and next_.ttype != T.Whitespace: 102 | tlist.insert_after(tidx, sql.Token(T.Whitespace, ' ')) 103 | 104 | pidx, prev_ = tlist.token_prev(tidx, skip_ws=False) 105 | if prev_ and prev_.ttype != T.Whitespace: 106 | tlist.insert_before(tidx, sql.Token(T.Whitespace, ' ')) 107 | tidx += 1 # has to shift since token inserted before it 108 | 109 | # assert tlist.token_index(token) == tidx 110 | tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) 111 | 112 | def process(self, stmt): 113 | [self.process(sgroup) for sgroup in stmt.get_sublists()] 114 | SpacesAroundOperatorsFilter._process(stmt) 115 | return stmt 116 | 117 | 118 | # --------------------------- 119 | # postprocess 120 | 121 | class SerializerUnicode(object): 122 | @staticmethod 123 | def process(stmt): 124 | lines = split_unquoted_newlines(stmt) 125 | return '\n'.join(line.rstrip() for line in lines) 126 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/output.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import sql, tokens as T 10 | from compat import text_type 11 | 12 | 13 | class OutputFilter(object): 14 | varname_prefix = '' 15 | 16 | def __init__(self, varname='sql'): 17 | self.varname = self.varname_prefix + varname 18 | self.count = 0 19 | 20 | def _process(self, stream, varname, has_nl): 21 | raise NotImplementedError 22 | 23 | def process(self, stmt): 24 | self.count += 1 25 | if self.count > 1: 26 | varname = u'{f.varname}{f.count}'.format(f=self) 27 | else: 28 | varname = self.varname 29 | 30 | has_nl = len(text_type(stmt).strip().splitlines()) > 1 31 | stmt.tokens = self._process(stmt.tokens, varname, has_nl) 32 | return stmt 33 | 34 | 35 | class OutputPythonFilter(OutputFilter): 36 | def _process(self, stream, varname, has_nl): 37 | # SQL query assignation to varname 38 | if self.count > 1: 39 | yield sql.Token(T.Whitespace, '\n') 40 | yield sql.Token(T.Name, varname) 41 | yield sql.Token(T.Whitespace, ' ') 42 | yield sql.Token(T.Operator, '=') 43 | yield sql.Token(T.Whitespace, ' ') 44 | if has_nl: 45 | yield sql.Token(T.Operator, '(') 46 | yield sql.Token(T.Text, "'") 47 | 48 | # Print the tokens on the quote 49 | for token in stream: 50 | # Token is a new line separator 51 | if token.is_whitespace and '\n' in token.value: 52 | # Close quote and add a new line 53 | yield sql.Token(T.Text, " '") 54 | yield sql.Token(T.Whitespace, '\n') 55 | 56 | # Quote header on secondary lines 57 | yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4)) 58 | yield sql.Token(T.Text, "'") 59 | 60 | # Indentation 61 | after_lb = token.value.split('\n', 1)[1] 62 | if after_lb: 63 | yield sql.Token(T.Whitespace, after_lb) 64 | continue 65 | 66 | # Token has escape chars 67 | elif "'" in token.value: 68 | token.value = token.value.replace("'", "\\'") 69 | 70 | # Put the token 71 | yield sql.Token(T.Text, token.value) 72 | 73 | # Close quote 74 | yield sql.Token(T.Text, "'") 75 | if has_nl: 76 | yield sql.Token(T.Operator, ')') 77 | 78 | 79 | class OutputPHPFilter(OutputFilter): 80 | varname_prefix = '$' 81 | 82 | def _process(self, stream, varname, has_nl): 83 | # SQL query assignation to varname (quote header) 84 | if self.count > 1: 85 | yield sql.Token(T.Whitespace, '\n') 86 | yield sql.Token(T.Name, varname) 87 | yield sql.Token(T.Whitespace, ' ') 88 | if has_nl: 89 | yield sql.Token(T.Whitespace, ' ') 90 | yield sql.Token(T.Operator, '=') 91 | yield sql.Token(T.Whitespace, ' ') 92 | yield sql.Token(T.Text, '"') 93 | 94 | # Print the tokens on the quote 95 | for token in stream: 96 | # Token is a new line separator 97 | if token.is_whitespace and '\n' in token.value: 98 | # Close quote and add a new line 99 | yield sql.Token(T.Text, ' ";') 100 | yield sql.Token(T.Whitespace, '\n') 101 | 102 | # Quote header on secondary lines 103 | yield sql.Token(T.Name, varname) 104 | yield sql.Token(T.Whitespace, ' ') 105 | yield sql.Token(T.Operator, '.=') 106 | yield sql.Token(T.Whitespace, ' ') 107 | yield sql.Token(T.Text, '"') 108 | 109 | # Indentation 110 | after_lb = token.value.split('\n', 1)[1] 111 | if after_lb: 112 | yield sql.Token(T.Whitespace, after_lb) 113 | continue 114 | 115 | # Token has escape chars 116 | elif '"' in token.value: 117 | token.value = token.value.replace('"', '\\"') 118 | 119 | # Put the token 120 | yield sql.Token(T.Text, token.value) 121 | 122 | # Close quote 123 | yield sql.Token(T.Text, '"') 124 | yield sql.Token(T.Punctuation, ';') 125 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/right_margin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import re 10 | 11 | import sql, tokens as T 12 | from compat import text_type 13 | 14 | 15 | # FIXME: Doesn't work 16 | class RightMarginFilter(object): 17 | keep_together = ( 18 | # sql.TypeCast, sql.Identifier, sql.Alias, 19 | ) 20 | 21 | def __init__(self, width=79): 22 | self.width = width 23 | self.line = '' 24 | 25 | def _process(self, group, stream): 26 | for token in stream: 27 | if token.is_whitespace and '\n' in token.value: 28 | if token.value.endswith('\n'): 29 | self.line = '' 30 | else: 31 | self.line = token.value.splitlines()[-1] 32 | elif token.is_group and type(token) not in self.keep_together: 33 | token.tokens = self._process(token, token.tokens) 34 | else: 35 | val = text_type(token) 36 | if len(self.line) + len(val) > self.width: 37 | match = re.search(r'^ +', self.line) 38 | if match is not None: 39 | indent = match.group() 40 | else: 41 | indent = '' 42 | yield sql.Token(T.Whitespace, '\n{0}'.format(indent)) 43 | self.line = indent 44 | self.line += val 45 | yield token 46 | 47 | def process(self, group): 48 | # return 49 | # group.tokens = self._process(group, group.tokens) 50 | raise NotImplementedError 51 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/statement_splitter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import sql, tokens as T 10 | 11 | 12 | class StatementSplitter(object): 13 | """Filter that split stream at individual statements""" 14 | 15 | def __init__(self): 16 | self._reset() 17 | 18 | def _reset(self): 19 | """Set the filter attributes to its default values""" 20 | self._in_declare = False 21 | self._is_create = False 22 | self._begin_depth = 0 23 | 24 | self.consume_ws = False 25 | self.tokens = [] 26 | self.level = 0 27 | 28 | def _change_splitlevel(self, ttype, value): 29 | """Get the new split level (increase, decrease or remain equal)""" 30 | 31 | # parenthesis increase/decrease a level 32 | if ttype is T.Punctuation and value == '(': 33 | return 1 34 | elif ttype is T.Punctuation and value == ')': 35 | return -1 36 | elif ttype not in T.Keyword: # if normal token return 37 | return 0 38 | 39 | # Everything after here is ttype = T.Keyword 40 | # Also to note, once entered an If statement you are done and basically 41 | # returning 42 | unified = value.upper() 43 | 44 | # three keywords begin with CREATE, but only one of them is DDL 45 | # DDL Create though can contain more words such as "or replace" 46 | if ttype is T.Keyword.DDL and unified.startswith('CREATE'): 47 | self._is_create = True 48 | return 0 49 | 50 | # can have nested declare inside of being... 51 | if unified == 'DECLARE' and self._is_create and self._begin_depth == 0: 52 | self._in_declare = True 53 | return 1 54 | 55 | if unified == 'BEGIN': 56 | self._begin_depth += 1 57 | if self._is_create: 58 | # FIXME(andi): This makes no sense. 59 | return 1 60 | return 0 61 | 62 | # Should this respect a preceding BEGIN? 63 | # In CASE ... WHEN ... END this results in a split level -1. 64 | # Would having multiple CASE WHEN END and a Assignment Operator 65 | # cause the statement to cut off prematurely? 66 | if unified == 'END': 67 | self._begin_depth = max(0, self._begin_depth - 1) 68 | return -1 69 | 70 | if (unified in ('IF', 'FOR', 'WHILE') 71 | and self._is_create and self._begin_depth > 0): 72 | return 1 73 | 74 | if unified in ('END IF', 'END FOR', 'END WHILE'): 75 | return -1 76 | 77 | # Default 78 | return 0 79 | 80 | def process(self, stream): 81 | """Process the stream""" 82 | EOS_TTYPE = T.Whitespace, T.Comment.Single 83 | 84 | # Run over all stream tokens 85 | for ttype, value in stream: 86 | # Yield token if we finished a statement and there's no whitespaces 87 | # It will count newline token as a non whitespace. In this context 88 | # whitespace ignores newlines. 89 | # why don't multi line comments also count? 90 | if self.consume_ws and ttype not in EOS_TTYPE: 91 | yield sql.Statement(self.tokens) 92 | 93 | # Reset filter and prepare to process next statement 94 | self._reset() 95 | 96 | # Change current split level (increase, decrease or remain equal) 97 | self.level += self._change_splitlevel(ttype, value) 98 | 99 | # Append the token to the current statement 100 | self.tokens.append(sql.Token(ttype, value)) 101 | 102 | # Check if we get the end of a statement 103 | if self.level <= 0 and ttype is T.Punctuation and value == ';': 104 | self.consume_ws = True 105 | 106 | # Yield pending statement (if any) 107 | if self.tokens: 108 | yield sql.Statement(self.tokens) 109 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/tokens.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | # 9 | # The Token implementation is based on pygment's token system written 10 | # by Georg Brandl. 11 | # http://pygments.org/ 12 | 13 | """Tokens""" 14 | 15 | 16 | class _TokenType(tuple): 17 | parent = None 18 | 19 | def __contains__(self, item): 20 | return item is not None and (self is item or item[:len(self)] == self) 21 | 22 | def __getattr__(self, name): 23 | new = _TokenType(self + (name,)) 24 | setattr(self, name, new) 25 | new.parent = self 26 | return new 27 | 28 | def __repr__(self): 29 | # self can be False only if its the `root` i.e. Token itself 30 | return 'Token' + ('.' if self else '') + '.'.join(self) 31 | 32 | 33 | Token = _TokenType() 34 | 35 | # Special token types 36 | Text = Token.Text 37 | Whitespace = Text.Whitespace 38 | Newline = Whitespace.Newline 39 | Error = Token.Error 40 | # Text that doesn't belong to this lexer (e.g. HTML in PHP) 41 | Other = Token.Other 42 | 43 | # Common token types for source code 44 | Keyword = Token.Keyword 45 | Name = Token.Name 46 | Literal = Token.Literal 47 | String = Literal.String 48 | Number = Literal.Number 49 | Punctuation = Token.Punctuation 50 | Operator = Token.Operator 51 | Comparison = Operator.Comparison 52 | Wildcard = Token.Wildcard 53 | Comment = Token.Comment 54 | Assignment = Token.Assignment 55 | 56 | # Generic types for non-source code 57 | Generic = Token.Generic 58 | Command = Generic.Command 59 | 60 | # String and some others are not direct children of Token. 61 | # alias them: 62 | Token.Token = Token 63 | Token.String = String 64 | Token.Number = Number 65 | 66 | # SQL specific tokens 67 | DML = Keyword.DML 68 | DDL = Keyword.DDL 69 | CTE = Keyword.CTE 70 | -------------------------------------------------------------------------------- /bring2lite/sqlparse/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2009-2018 the sqlparse authors and contributors 4 | # 5 | # 6 | # This module is part of python-sqlparse and is released under 7 | # the BSD License: https://opensource.org/licenses/BSD-3-Clause 8 | 9 | import itertools 10 | import re 11 | from collections import deque 12 | from contextlib import contextmanager 13 | from compat import text_type 14 | 15 | # This regular expression replaces the home-cooked parser that was here before. 16 | # It is much faster, but requires an extra post-processing step to get the 17 | # desired results (that are compatible with what you would expect from the 18 | # str.splitlines() method). 19 | # 20 | # It matches groups of characters: newlines, quoted strings, or unquoted text, 21 | # and splits on that basis. The post-processing step puts those back together 22 | # into the actual lines of SQL. 23 | SPLIT_REGEX = re.compile(r""" 24 | ( 25 | (?: # Start of non-capturing group 26 | (?:\r\n|\r|\n) | # Match any single newline, or 27 | [^\r\n'"]+ | # Match any character series without quotes or 28 | # newlines, or 29 | "(?:[^"\\]|\\.)*" | # Match double-quoted strings, or 30 | '(?:[^'\\]|\\.)*' # Match single quoted strings 31 | ) 32 | ) 33 | """, re.VERBOSE) 34 | 35 | LINE_MATCH = re.compile(r'(\r\n|\r|\n)') 36 | 37 | 38 | def split_unquoted_newlines(stmt): 39 | """Split a string on all unquoted newlines. 40 | 41 | Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite 42 | character is inside of a string.""" 43 | text = text_type(stmt) 44 | lines = SPLIT_REGEX.split(text) 45 | outputlines = [''] 46 | for line in lines: 47 | if not line: 48 | continue 49 | elif LINE_MATCH.match(line): 50 | outputlines.append('') 51 | else: 52 | outputlines[-1] += line 53 | return outputlines 54 | 55 | 56 | def remove_quotes(val): 57 | """Helper that removes surrounding quotes from strings.""" 58 | if val is None: 59 | return 60 | if val[0] in ('"', "'") and val[0] == val[-1]: 61 | val = val[1:-1] 62 | return val 63 | 64 | 65 | def recurse(*cls): 66 | """Function decorator to help with recursion 67 | 68 | :param cls: Classes to not recurse over 69 | :return: function 70 | """ 71 | def wrap(f): 72 | def wrapped_f(tlist): 73 | for sgroup in tlist.get_sublists(): 74 | if not isinstance(sgroup, cls): 75 | wrapped_f(sgroup) 76 | f(tlist) 77 | 78 | return wrapped_f 79 | 80 | return wrap 81 | 82 | 83 | def imt(token, i=None, m=None, t=None): 84 | """Helper function to simplify comparisons Instance, Match and TokenType 85 | :param token: 86 | :param i: Class or Tuple/List of Classes 87 | :param m: Tuple of TokenType & Value. Can be list of Tuple for multiple 88 | :param t: TokenType or Tuple/List of TokenTypes 89 | :return: bool 90 | """ 91 | clss = i 92 | types = [t, ] if t and not isinstance(t, list) else t 93 | mpatterns = [m, ] if m and not isinstance(m, list) else m 94 | 95 | if token is None: 96 | return False 97 | elif clss and isinstance(token, clss): 98 | return True 99 | elif mpatterns and any(token.match(*pattern) for pattern in mpatterns): 100 | return True 101 | elif types and any(token.ttype in ttype for ttype in types): 102 | return True 103 | else: 104 | return False 105 | 106 | 107 | def consume(iterator, n): 108 | """Advance the iterator n-steps ahead. If n is none, consume entirely.""" 109 | deque(itertools.islice(iterator, n), maxlen=0) 110 | 111 | 112 | @contextmanager 113 | def offset(filter_, n=0): 114 | filter_.offset += n 115 | yield 116 | filter_.offset -= n 117 | 118 | 119 | @contextmanager 120 | def indent(filter_, n=1): 121 | filter_.indent += n 122 | yield 123 | filter_.indent -= n 124 | -------------------------------------------------------------------------------- /bring2lite/tqdm/LICENSE: -------------------------------------------------------------------------------- 1 | `tqdm` is a product of collaborative work. 2 | Unless otherwise stated, all authors (see commit logs) retain copyright 3 | for their respective work, and release the work under the MIT licence 4 | (text below). 5 | 6 | Exceptions or notable authors are listed below 7 | in reverse chronological order: 8 | 9 | * files: * 10 | MPLv2.0 2015-2020 (c) Casper da Costa-Luis 11 | [casperdcl](https://github.com/casperdcl). 12 | * files: tqdm/_tqdm.py 13 | MIT 2016 (c) [PR #96] on behalf of Google Inc. 14 | * files: tqdm/_tqdm.py setup.py README.rst MANIFEST.in .gitignore 15 | MIT 2013 (c) Noam Yorav-Raphael, original author. 16 | 17 | [PR #96]: https://github.com/tqdm/tqdm/pull/96 18 | 19 | 20 | Mozilla Public Licence (MPL) v. 2.0 - Exhibit A 21 | ----------------------------------------------- 22 | 23 | This Source Code Form is subject to the terms of the 24 | Mozilla Public License, v. 2.0. 25 | If a copy of the MPL was not distributed with this file, 26 | You can obtain one at https://mozilla.org/MPL/2.0/. 27 | 28 | 29 | MIT License (MIT) 30 | ----------------- 31 | 32 | Copyright (c) 2013 noamraph 33 | 34 | Permission is hereby granted, free of charge, to any person obtaining a copy of 35 | this software and associated documentation files (the "Software"), to deal in 36 | the Software without restriction, including without limitation the rights to 37 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 38 | the Software, and to permit persons to whom the Software is furnished to do so, 39 | subject to the following conditions: 40 | 41 | The above copyright notice and this permission notice shall be included in all 42 | copies or substantial portions of the Software. 43 | 44 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 45 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 46 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 47 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 48 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 49 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /bring2lite/tqdm/__init__.py: -------------------------------------------------------------------------------- 1 | from .std import tqdm, trange 2 | from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0 3 | from .gui import trange as tgrange # TODO: remove in v5.0.0 4 | from ._tqdm_pandas import tqdm_pandas 5 | from .cli import main # TODO: remove in v5.0.0 6 | from ._monitor import TMonitor, TqdmSynchronisationWarning 7 | from ._version import __version__ # NOQA 8 | from .std import TqdmTypeError, TqdmKeyError, TqdmWarning, \ 9 | TqdmDeprecationWarning, TqdmExperimentalWarning, \ 10 | TqdmMonitorWarning 11 | 12 | __all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas', 13 | 'tqdm_notebook', 'tnrange', 'main', 'TMonitor', 14 | 'TqdmTypeError', 'TqdmKeyError', 15 | 'TqdmWarning', 'TqdmDeprecationWarning', 16 | 'TqdmExperimentalWarning', 17 | 'TqdmMonitorWarning', 'TqdmSynchronisationWarning', 18 | '__version__'] 19 | 20 | 21 | def tqdm_notebook(*args, **kwargs): # pragma: no cover 22 | """See tqdm.notebook.tqdm for full documentation""" 23 | from .notebook import tqdm as _tqdm_notebook 24 | from warnings import warn 25 | warn("This function will be removed in tqdm==5.0.0\n" 26 | "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`", 27 | TqdmDeprecationWarning, stacklevel=2) 28 | return _tqdm_notebook(*args, **kwargs) 29 | 30 | 31 | def tnrange(*args, **kwargs): # pragma: no cover 32 | """ 33 | A shortcut for `tqdm.notebook.tqdm(xrange(*args), **kwargs)`. 34 | On Python3+, `range` is used instead of `xrange`. 35 | """ 36 | from .notebook import trange as _tnrange 37 | from warnings import warn 38 | warn("Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`", 39 | TqdmDeprecationWarning, stacklevel=2) 40 | return _tnrange(*args, **kwargs) 41 | -------------------------------------------------------------------------------- /bring2lite/tqdm/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli import main 2 | main() 3 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_main.py: -------------------------------------------------------------------------------- 1 | from .cli import * # NOQA 2 | from .cli import __all__ # NOQA 3 | from .std import TqdmDeprecationWarning 4 | from warnings import warn 5 | warn("This function will be removed in tqdm==5.0.0\n" 6 | "Please use `tqdm.cli.*` instead of `tqdm._main.*`", 7 | TqdmDeprecationWarning, stacklevel=2) 8 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_monitor.py: -------------------------------------------------------------------------------- 1 | from threading import Event, Thread, current_thread 2 | from time import time 3 | from warnings import warn 4 | import atexit 5 | __all__ = ["TMonitor", "TqdmSynchronisationWarning"] 6 | 7 | 8 | class TqdmSynchronisationWarning(RuntimeWarning): 9 | """tqdm multi-thread/-process errors which may cause incorrect nesting 10 | but otherwise no adverse effects""" 11 | pass 12 | 13 | 14 | class TMonitor(Thread): 15 | """ 16 | Monitoring thread for tqdm bars. 17 | Monitors if tqdm bars are taking too much time to display 18 | and readjusts miniters automatically if necessary. 19 | 20 | Parameters 21 | ---------- 22 | tqdm_cls : class 23 | tqdm class to use (can be core tqdm or a submodule). 24 | sleep_interval : fload 25 | Time to sleep between monitoring checks. 26 | """ 27 | 28 | # internal vars for unit testing 29 | _time = None 30 | _event = None 31 | 32 | def __init__(self, tqdm_cls, sleep_interval): 33 | Thread.__init__(self) 34 | self.daemon = True # kill thread when main killed (KeyboardInterrupt) 35 | self.was_killed = Event() 36 | self.woken = 0 # last time woken up, to sync with monitor 37 | self.tqdm_cls = tqdm_cls 38 | self.sleep_interval = sleep_interval 39 | if TMonitor._time is not None: 40 | self._time = TMonitor._time 41 | else: 42 | self._time = time 43 | if TMonitor._event is not None: 44 | self._event = TMonitor._event 45 | else: 46 | self._event = Event 47 | atexit.register(self.exit) 48 | self.start() 49 | 50 | def exit(self): 51 | self.was_killed.set() 52 | if self is not current_thread(): 53 | self.join() 54 | return self.report() 55 | 56 | def get_instances(self): 57 | # returns a copy of started `tqdm_cls` instances 58 | return [i for i in self.tqdm_cls._instances.copy() 59 | # Avoid race by checking that the instance started 60 | if hasattr(i, 'start_t')] 61 | 62 | def run(self): 63 | cur_t = self._time() 64 | while True: 65 | # After processing and before sleeping, notify that we woke 66 | # Need to be done just before sleeping 67 | self.woken = cur_t 68 | # Sleep some time... 69 | self.was_killed.wait(self.sleep_interval) 70 | # Quit if killed 71 | if self.was_killed.is_set(): 72 | return 73 | # Then monitor! 74 | # Acquire lock (to access _instances) 75 | with self.tqdm_cls.get_lock(): 76 | cur_t = self._time() 77 | # Check tqdm instances are waiting too long to print 78 | instances = self.get_instances() 79 | for instance in instances: 80 | # Check event in loop to reduce blocking time on exit 81 | if self.was_killed.is_set(): 82 | return 83 | # Only if mininterval > 1 (else iterations are just slow) 84 | # and last refresh exceeded maxinterval 85 | if instance.miniters > 1 and \ 86 | (cur_t - instance.last_print_t) >= \ 87 | instance.maxinterval: 88 | # force bypassing miniters on next iteration 89 | # (dynamic_miniters adjusts mininterval automatically) 90 | instance.miniters = 1 91 | # Refresh now! (works only for manual tqdm) 92 | instance.refresh(nolock=True) 93 | # Remove accidental long-lived strong reference 94 | del instance 95 | if instances != self.get_instances(): # pragma: nocover 96 | warn("Set changed size during iteration" + 97 | " (see https://github.com/tqdm/tqdm/issues/481)", 98 | TqdmSynchronisationWarning, stacklevel=2) 99 | # Remove accidental long-lived strong references 100 | del instances 101 | 102 | def report(self): 103 | return not self.was_killed.is_set() 104 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_tqdm.py: -------------------------------------------------------------------------------- 1 | from .std import * # NOQA 2 | from .std import __all__ # NOQA 3 | from .std import TqdmDeprecationWarning 4 | from warnings import warn 5 | warn("This function will be removed in tqdm==5.0.0\n" 6 | "Please use `tqdm.std.*` instead of `tqdm._tqdm.*`", 7 | TqdmDeprecationWarning, stacklevel=2) 8 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_tqdm_gui.py: -------------------------------------------------------------------------------- 1 | from .gui import * # NOQA 2 | from .gui import __all__ # NOQA 3 | from .std import TqdmDeprecationWarning 4 | from warnings import warn 5 | warn("This function will be removed in tqdm==5.0.0\n" 6 | "Please use `tqdm.gui.*` instead of `tqdm._tqdm_gui.*`", 7 | TqdmDeprecationWarning, stacklevel=2) 8 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_tqdm_notebook.py: -------------------------------------------------------------------------------- 1 | from .notebook import * # NOQA 2 | from .notebook import __all__ # NOQA 3 | from .std import TqdmDeprecationWarning 4 | from warnings import warn 5 | warn("This function will be removed in tqdm==5.0.0\n" 6 | "Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`", 7 | TqdmDeprecationWarning, stacklevel=2) 8 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_tqdm_pandas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | __author__ = "github.com/casperdcl" 4 | __all__ = ['tqdm_pandas'] 5 | 6 | 7 | def tqdm_pandas(tclass, **tqdm_kwargs): 8 | """ 9 | Registers the given `tqdm` instance with 10 | `pandas.core.groupby.DataFrameGroupBy.progress_apply`. 11 | """ 12 | from tqdm import TqdmDeprecationWarning 13 | 14 | if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith( 15 | 'tqdm_')): # delayed adapter case 16 | TqdmDeprecationWarning("""\ 17 | Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`. 18 | """, fp_write=getattr(tqdm_kwargs.get('file', None), 'write', sys.stderr.write)) 19 | tclass.pandas(**tqdm_kwargs) 20 | else: 21 | TqdmDeprecationWarning("""\ 22 | Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`. 23 | """, fp_write=getattr(tclass.fp, 'write', sys.stderr.write)) 24 | type(tclass).pandas(deprecated_t=tclass) 25 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_utils.py: -------------------------------------------------------------------------------- 1 | from .utils import CUR_OS, IS_WIN, IS_NIX, RE_ANSI, _range, _unich, _unicode, colorama, WeakSet, _basestring, _OrderedDict, FormatReplace, Comparable, SimpleTextIOWrapper, _is_utf, _supports_unicode, _is_ascii, _screen_shape_wrapper, _screen_shape_windows, _screen_shape_tput, _screen_shape_linux, _environ_cols_wrapper, _term_move_up # NOQA 2 | from .std import TqdmDeprecationWarning 3 | from warnings import warn 4 | warn("This function will be removed in tqdm==5.0.0\n" 5 | "Please use `tqdm.utils.*` instead of `tqdm._utils.*`", 6 | TqdmDeprecationWarning, stacklevel=2) 7 | -------------------------------------------------------------------------------- /bring2lite/tqdm/_version.py: -------------------------------------------------------------------------------- 1 | # Definition of the version number 2 | import os 3 | import io 4 | 5 | __all__ = ["__version__"] 6 | 7 | # major, minor, patch, -extra 8 | version_info = 4, 48, 2 9 | 10 | # Nice string for the version 11 | __version__ = '.'.join(map(str, version_info)) 12 | 13 | 14 | # auto -extra based on commit hash (if not tagged as release) 15 | scriptdir = os.path.dirname(__file__) 16 | gitdir = os.path.abspath(os.path.join(scriptdir, "..", ".git")) 17 | if os.path.isdir(gitdir): # pragma: nocover 18 | try: 19 | extra = None 20 | # Open config file to check if we are in tqdm project 21 | with io.open(os.path.join(gitdir, "config"), 'r') as fh_config: 22 | if 'tqdm' in fh_config.read(): 23 | # Open the HEAD file 24 | with io.open(os.path.join(gitdir, "HEAD"), 'r') as fh_head: 25 | extra = fh_head.readline().strip() 26 | # in a branch => HEAD points to file containing last commit 27 | if 'ref:' in extra: 28 | # reference file path 29 | ref_file = extra[5:] 30 | branch_name = ref_file.rsplit('/', 1)[-1] 31 | 32 | ref_file_path = os.path.abspath(os.path.join( 33 | gitdir, ref_file)) 34 | # check that we are in git folder 35 | # (by stripping the git folder from the ref file path) 36 | if os.path.relpath(ref_file_path, gitdir).replace( 37 | '\\', '/') != ref_file: 38 | # out of git folder 39 | extra = None 40 | else: 41 | # open the ref file 42 | with io.open(ref_file_path, 'r') as fh_branch: 43 | commit_hash = fh_branch.readline().strip() 44 | extra = commit_hash[:8] 45 | if branch_name != "master": 46 | extra += '.' + branch_name 47 | 48 | # detached HEAD mode, already have commit hash 49 | else: 50 | extra = extra[:8] 51 | 52 | # Append commit hash (and branch) to version string if not tagged 53 | if extra is not None: 54 | with io.open(os.path.join(gitdir, "refs", "tags", 55 | 'v' + __version__)) as fdv: 56 | if fdv.readline().strip()[:8] != extra[:8]: 57 | __version__ += '-' + extra 58 | except Exception as e: 59 | if "No such file" in str(e): 60 | __version__ += "-git.UNKNOWN" 61 | else: 62 | raise 63 | -------------------------------------------------------------------------------- /bring2lite/tqdm/asyncio.py: -------------------------------------------------------------------------------- 1 | """ 2 | Asynchronous progressbar decorator for iterators. 3 | Includes a default `range` iterator printing to `stderr`. 4 | 5 | Usage: 6 | >>> from tqdm.asyncio import trange, tqdm 7 | >>> async for i in trange(10): 8 | ... ... 9 | """ 10 | from .std import tqdm as std_tqdm 11 | import asyncio 12 | __author__ = {"github.com/": ["casperdcl"]} 13 | __all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] 14 | 15 | 16 | class tqdm_asyncio(std_tqdm): 17 | """ 18 | Asynchronous-friendly version of tqdm (Python 3.5+). 19 | """ 20 | def __init__(self, iterable=None, *args, **kwargs): 21 | super(tqdm_asyncio, self).__init__(iterable, *args, **kwargs) 22 | self.iterable_awaitable = False 23 | if iterable is not None: 24 | if hasattr(iterable, "__anext__"): 25 | self.iterable_next = iterable.__anext__ 26 | self.iterable_awaitable = True 27 | elif hasattr(iterable, "__next__"): 28 | self.iterable_next = iterable.__next__ 29 | else: 30 | self.iterable_iterator = iter(iterable) 31 | self.iterable_next = self.iterable_iterator.__next__ 32 | 33 | def __aiter__(self): 34 | return self 35 | 36 | async def __anext__(self): 37 | try: 38 | if self.iterable_awaitable: 39 | res = await self.iterable_next() 40 | else: 41 | res = self.iterable_next() 42 | self.update() 43 | return res 44 | except StopIteration: 45 | self.close() 46 | raise StopAsyncIteration 47 | except: 48 | self.close() 49 | raise 50 | 51 | def send(self, *args, **kwargs): 52 | return self.iterable.send(*args, **kwargs) 53 | 54 | @classmethod 55 | def as_completed(cls, fs, *, loop=None, timeout=None, total=None, 56 | **tqdm_kwargs): 57 | """ 58 | Wrapper for `asyncio.as_completed`. 59 | """ 60 | if total is None: 61 | total = len(fs) 62 | yield from cls(asyncio.as_completed(fs, loop=loop, timeout=timeout), 63 | total=total, **tqdm_kwargs) 64 | 65 | 66 | def tarange(*args, **kwargs): 67 | """ 68 | A shortcut for `tqdm.asyncio.tqdm(range(*args), **kwargs)`. 69 | """ 70 | return tqdm_asyncio(range(*args), **kwargs) 71 | 72 | 73 | # Aliases 74 | tqdm = tqdm_asyncio 75 | trange = tarange 76 | -------------------------------------------------------------------------------- /bring2lite/tqdm/auto.py: -------------------------------------------------------------------------------- 1 | """ 2 | Enables multiple commonly used features. 3 | 4 | Method resolution order: 5 | 6 | - `tqdm.autonotebook` without import warnings 7 | - `tqdm.asyncio` on Python3.5+ 8 | - `tqdm.std` base class 9 | 10 | Usage: 11 | >>> from tqdm.auto import trange, tqdm 12 | >>> for i in trange(10): 13 | ... ... 14 | """ 15 | import sys 16 | import warnings 17 | from .std import TqdmExperimentalWarning 18 | with warnings.catch_warnings(): 19 | warnings.simplefilter("ignore", category=TqdmExperimentalWarning) 20 | from .autonotebook import tqdm as notebook_tqdm 21 | from .autonotebook import trange as notebook_trange 22 | 23 | if sys.version_info[:1] < (3, 4): 24 | tqdm = notebook_tqdm 25 | trange = notebook_trange 26 | else: # Python3.5+ 27 | from .asyncio import tqdm as asyncio_tqdm 28 | from .std import tqdm as std_tqdm 29 | 30 | if notebook_tqdm != std_tqdm: 31 | class tqdm(notebook_tqdm, asyncio_tqdm): 32 | pass 33 | else: 34 | tqdm = asyncio_tqdm 35 | 36 | def trange(*args, **kwargs): 37 | """ 38 | A shortcut for `tqdm.auto.tqdm(range(*args), **kwargs)`. 39 | """ 40 | return tqdm(range(*args), **kwargs) 41 | 42 | __all__ = ["tqdm", "trange"] 43 | -------------------------------------------------------------------------------- /bring2lite/tqdm/autonotebook.py: -------------------------------------------------------------------------------- 1 | """ 2 | Automatically choose between `tqdm.notebook` and `tqdm.std`. 3 | 4 | Usage: 5 | >>> from tqdm.autonotebook import trange, tqdm 6 | >>> for i in trange(10): 7 | ... ... 8 | """ 9 | import os 10 | import sys 11 | 12 | try: 13 | get_ipython = sys.modules['IPython'].get_ipython 14 | if 'IPKernelApp' not in get_ipython().config: # pragma: no cover 15 | raise ImportError("console") 16 | if 'VSCODE_PID' in os.environ: # pragma: no cover 17 | raise ImportError("vscode") 18 | except: 19 | from .std import tqdm, trange 20 | else: # pragma: no cover 21 | from .notebook import tqdm, trange 22 | from .std import TqdmExperimentalWarning 23 | from warnings import warn 24 | warn("Using `tqdm.autonotebook.tqdm` in notebook mode." 25 | " Use `tqdm.tqdm` instead to force console mode" 26 | " (e.g. in jupyter console)", TqdmExperimentalWarning, stacklevel=2) 27 | __all__ = ["tqdm", "trange"] 28 | -------------------------------------------------------------------------------- /bring2lite/tqdm/completion.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | _tqdm(){ 3 | local cur prv 4 | cur="${COMP_WORDS[COMP_CWORD]}" 5 | prv="${COMP_WORDS[COMP_CWORD - 1]}" 6 | 7 | case ${prv} in 8 | --bar_format|--buf_size|--comppath|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor) 9 | # await user input 10 | ;; 11 | "--log") 12 | COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur})) 13 | ;; 14 | *) 15 | COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --comppath --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --position --postfix --smoothing --total --unit --unit_divisor --unit_scale --version --write_bytes -h -v' -- ${cur})) 16 | ;; 17 | esac 18 | } 19 | complete -F _tqdm tqdm 20 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around common functions. 3 | 4 | Subpackages contain potentially unstable extensions. 5 | """ 6 | from tqdm import tqdm 7 | from tqdm.auto import tqdm as tqdm_auto 8 | from tqdm.utils import ObjectWrapper 9 | from functools import wraps 10 | import sys 11 | __author__ = {"github.com/": ["casperdcl"]} 12 | __all__ = ['tenumerate', 'tzip', 'tmap'] 13 | 14 | 15 | class DummyTqdmFile(ObjectWrapper): 16 | """Dummy file-like that will write to tqdm""" 17 | def write(self, x, nolock=False): 18 | # Avoid print() second call (useless \n) 19 | if len(x.rstrip()) > 0: 20 | tqdm.write(x, file=self._wrapped, nolock=nolock) 21 | 22 | 23 | def builtin_iterable(func): 24 | """Wraps `func()` output in a `list()` in py2""" 25 | if sys.version_info[:1] < (3,): 26 | @wraps(func) 27 | def inner(*args, **kwargs): 28 | return list(func(*args, **kwargs)) 29 | return inner 30 | return func 31 | 32 | 33 | def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, 34 | **tqdm_kwargs): 35 | """ 36 | Equivalent of `numpy.ndenumerate` or builtin `enumerate`. 37 | 38 | Parameters 39 | ---------- 40 | tqdm_class : [default: tqdm.auto.tqdm]. 41 | """ 42 | try: 43 | import numpy as np 44 | except ImportError: 45 | pass 46 | else: 47 | if isinstance(iterable, np.ndarray): 48 | return tqdm_class(np.ndenumerate(iterable), 49 | total=total or iterable.size, **tqdm_kwargs) 50 | return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start) 51 | 52 | 53 | @builtin_iterable 54 | def tzip(iter1, *iter2plus, **tqdm_kwargs): 55 | """ 56 | Equivalent of builtin `zip`. 57 | 58 | Parameters 59 | ---------- 60 | tqdm_class : [default: tqdm.auto.tqdm]. 61 | """ 62 | kwargs = tqdm_kwargs.copy() 63 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 64 | for i in zip(tqdm_class(iter1, **tqdm_kwargs), *iter2plus): 65 | yield i 66 | 67 | 68 | @builtin_iterable 69 | def tmap(function, *sequences, **tqdm_kwargs): 70 | """ 71 | Equivalent of builtin `map`. 72 | 73 | Parameters 74 | ---------- 75 | tqdm_class : [default: tqdm.auto.tqdm]. 76 | """ 77 | for i in tzip(*sequences, **tqdm_kwargs): 78 | yield function(*i) 79 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/bells.py: -------------------------------------------------------------------------------- 1 | """ 2 | Even more features than `tqdm.auto` (all the bells & whistles): 3 | 4 | - `tqdm.auto` 5 | - `tqdm.tqdm.pandas` 6 | - `tqdm.contrib.telegram` 7 | + uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}` 8 | - `tqdm.contrib.discord` 9 | + uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}` 10 | """ 11 | __all__ = ['tqdm', 'trange'] 12 | from os import getenv 13 | import warnings 14 | 15 | 16 | if getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"): 17 | from tqdm.contrib.telegram import tqdm, trange 18 | elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"): 19 | from tqdm.contrib.discord import tqdm, trange 20 | else: 21 | from tqdm.auto import tqdm, trange 22 | 23 | with warnings.catch_warnings(): 24 | warnings.simplefilter("ignore", category=FutureWarning) 25 | tqdm.pandas() 26 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/concurrent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around `concurrent.futures`. 3 | """ 4 | from __future__ import absolute_import 5 | from tqdm import TqdmWarning 6 | from tqdm.auto import tqdm as tqdm_auto 7 | try: 8 | from operator import length_hint 9 | except ImportError: 10 | def length_hint(it, default=0): 11 | """Returns `len(it)`, falling back to `default`""" 12 | try: 13 | return len(it) 14 | except TypeError: 15 | return default 16 | try: 17 | from os import cpu_count 18 | except ImportError: 19 | try: 20 | from multiprocessing import cpu_count 21 | except ImportError: 22 | def cpu_count(): 23 | return 4 24 | import sys 25 | __author__ = {"github.com/": ["casperdcl"]} 26 | __all__ = ['thread_map', 'process_map'] 27 | 28 | 29 | def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs): 30 | """ 31 | Implementation of `thread_map` and `process_map`. 32 | 33 | Parameters 34 | ---------- 35 | tqdm_class : [default: tqdm.auto.tqdm]. 36 | max_workers : [default: min(32, cpu_count() + 4)]. 37 | chunksize : [default: 1]. 38 | """ 39 | kwargs = tqdm_kwargs.copy() 40 | if "total" not in kwargs: 41 | kwargs["total"] = len(iterables[0]) 42 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 43 | max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4)) 44 | chunksize = kwargs.pop("chunksize", 1) 45 | pool_kwargs = dict(max_workers=max_workers) 46 | sys_version = sys.version_info[:2] 47 | if sys_version >= (3, 7): 48 | # share lock in case workers are already using `tqdm` 49 | pool_kwargs.update( 50 | initializer=tqdm_class.set_lock, initargs=(tqdm_class.get_lock(),)) 51 | map_args = {} 52 | if not (3, 0) < sys_version < (3, 5): 53 | map_args.update(chunksize=chunksize) 54 | with PoolExecutor(**pool_kwargs) as ex: 55 | return list(tqdm_class( 56 | ex.map(fn, *iterables, **map_args), **kwargs)) 57 | 58 | 59 | def thread_map(fn, *iterables, **tqdm_kwargs): 60 | """ 61 | Equivalent of `list(map(fn, *iterables))` 62 | driven by `concurrent.futures.ThreadPoolExecutor`. 63 | 64 | Parameters 65 | ---------- 66 | tqdm_class : optional 67 | `tqdm` class to use for bars [default: tqdm.auto.tqdm]. 68 | max_workers : int, optional 69 | Maximum number of workers to spawn; passed to 70 | `concurrent.futures.ThreadPoolExecutor.__init__`. 71 | [default: max(32, cpu_count() + 4)]. 72 | """ 73 | from concurrent.futures import ThreadPoolExecutor 74 | return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) 75 | 76 | 77 | def process_map(fn, *iterables, **tqdm_kwargs): 78 | """ 79 | Equivalent of `list(map(fn, *iterables))` 80 | driven by `concurrent.futures.ProcessPoolExecutor`. 81 | 82 | Parameters 83 | ---------- 84 | tqdm_class : optional 85 | `tqdm` class to use for bars [default: tqdm.auto.tqdm]. 86 | max_workers : int, optional 87 | Maximum number of workers to spawn; passed to 88 | `concurrent.futures.ProcessPoolExecutor.__init__`. 89 | [default: min(32, cpu_count() + 4)]. 90 | chunksize : int, optional 91 | Size of chunks sent to worker processes; passed to 92 | `concurrent.futures.ProcessPoolExecutor.map`. [default: 1]. 93 | """ 94 | from concurrent.futures import ProcessPoolExecutor 95 | if iterables and "chunksize" not in tqdm_kwargs: 96 | # default `chunksize=1` has poor performance for large iterables 97 | # (most time spent dispatching items to workers). 98 | longest_iterable_len = max(map(length_hint, iterables)) 99 | if longest_iterable_len > 1000: 100 | from warnings import warn 101 | warn("Iterable length %d > 1000 but `chunksize` is not set." 102 | " This may seriously degrade multiprocess performance." 103 | " Set `chunksize=1` or more." % longest_iterable_len, 104 | TqdmWarning, stacklevel=2) 105 | return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs) 106 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/discord.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends updates to a Discord bot. 3 | 4 | Usage: 5 | >>> from tqdm.contrib.discord import tqdm, trange 6 | >>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): 7 | ... ... 8 | 9 | ![screenshot]( 10 | https://raw.githubusercontent.com/tqdm/img/src/screenshot-discord.png) 11 | """ 12 | from __future__ import absolute_import 13 | import logging 14 | from os import getenv 15 | 16 | try: 17 | from disco.client import Client, ClientConfig 18 | except ImportError: 19 | raise ImportError("Please `pip install disco-py`") 20 | 21 | from tqdm.auto import tqdm as tqdm_auto 22 | from tqdm.utils import _range 23 | from .utils_worker import MonoWorker 24 | __author__ = {"github.com/": ["casperdcl"]} 25 | __all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange'] 26 | 27 | 28 | class DiscordIO(MonoWorker): 29 | """Non-blocking file-like IO using a Discord Bot.""" 30 | def __init__(self, token, channel_id): 31 | """Creates a new message in the given `channel_id`.""" 32 | super(DiscordIO, self).__init__() 33 | config = ClientConfig() 34 | config.token = token 35 | client = Client(config) 36 | self.text = self.__class__.__name__ 37 | try: 38 | self.message = client.api.channels_messages_create( 39 | channel_id, self.text) 40 | except Exception as e: 41 | tqdm_auto.write(str(e)) 42 | 43 | def write(self, s): 44 | """Replaces internal `message`'s text with `s`.""" 45 | if not s: 46 | return 47 | s = s.replace('\r', '').strip() 48 | if s == self.text: 49 | return # skip duplicate message 50 | self.text = s 51 | try: 52 | future = self.submit(self.message.edit, '`' + s + '`') 53 | except Exception as e: 54 | tqdm_auto.write(str(e)) 55 | else: 56 | return future 57 | 58 | 59 | class tqdm_discord(tqdm_auto): 60 | """ 61 | Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot. 62 | May take a few seconds to create (`__init__`). 63 | 64 | - create a discord bot (not public, no requirement of OAuth2 code 65 | grant, only send message permissions) & invite it to a channel: 66 | 67 | - copy the bot `{token}` & `{channel_id}` and paste below 68 | 69 | >>> from tqdm.contrib.discord import tqdm, trange 70 | >>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): 71 | ... ... 72 | """ 73 | def __init__(self, *args, **kwargs): 74 | """ 75 | Parameters 76 | ---------- 77 | token : str, required. Discord token 78 | [default: ${TQDM_DISCORD_TOKEN}]. 79 | channel_id : int, required. Discord channel ID 80 | [default: ${TQDM_DISCORD_CHANNEL_ID}]. 81 | mininterval : float, optional. 82 | Minimum of [default: 1.5] to avoid rate limit. 83 | 84 | See `tqdm.auto.tqdm.__init__` for other parameters. 85 | """ 86 | kwargs = kwargs.copy() 87 | logging.getLogger("HTTPClient").setLevel(logging.WARNING) 88 | self.dio = DiscordIO( 89 | kwargs.pop('token', getenv("TQDM_DISCORD_TOKEN")), 90 | kwargs.pop('channel_id', getenv("TQDM_DISCORD_CHANNEL_ID"))) 91 | 92 | kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) 93 | super(tqdm_discord, self).__init__(*args, **kwargs) 94 | 95 | def display(self, **kwargs): 96 | super(tqdm_discord, self).display(**kwargs) 97 | fmt = self.format_dict 98 | if 'bar_format' in fmt and fmt['bar_format']: 99 | fmt['bar_format'] = fmt['bar_format'].replace('', '{bar}') 100 | else: 101 | fmt['bar_format'] = '{l_bar}{bar}{r_bar}' 102 | fmt['bar_format'] = fmt['bar_format'].replace('{bar}', '{bar:10u}') 103 | self.dio.write(self.format_meter(**fmt)) 104 | 105 | def __new__(cls, *args, **kwargs): 106 | """ 107 | Workaround for mixed-class same-stream nested progressbars. 108 | See [#509](https://github.com/tqdm/tqdm/issues/509) 109 | """ 110 | with cls.get_lock(): 111 | try: 112 | cls._instances = tqdm_auto._instances 113 | except AttributeError: 114 | pass 115 | instance = super(tqdm_discord, cls).__new__(cls, *args, **kwargs) 116 | with cls.get_lock(): 117 | try: 118 | # `tqdm_auto` may have been changed so update 119 | cls._instances.update(tqdm_auto._instances) 120 | except AttributeError: 121 | pass 122 | tqdm_auto._instances = cls._instances 123 | return instance 124 | 125 | 126 | def tdrange(*args, **kwargs): 127 | """ 128 | A shortcut for `tqdm.contrib.discord.tqdm(xrange(*args), **kwargs)`. 129 | On Python3+, `range` is used instead of `xrange`. 130 | """ 131 | return tqdm_discord(_range(*args), **kwargs) 132 | 133 | 134 | # Aliases 135 | tqdm = tqdm_discord 136 | trange = tdrange 137 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/itertools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around `itertools`. 3 | """ 4 | from __future__ import absolute_import 5 | from tqdm.auto import tqdm as tqdm_auto 6 | import itertools 7 | __author__ = {"github.com/": ["casperdcl"]} 8 | __all__ = ['product'] 9 | 10 | 11 | def product(*iterables, **tqdm_kwargs): 12 | """ 13 | Equivalent of `itertools.product`. 14 | 15 | Parameters 16 | ---------- 17 | tqdm_class : [default: tqdm.auto.tqdm]. 18 | """ 19 | kwargs = tqdm_kwargs.copy() 20 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 21 | try: 22 | lens = list(map(len, iterables)) 23 | except TypeError: 24 | total = None 25 | else: 26 | total = 1 27 | for i in lens: 28 | total *= i 29 | kwargs.setdefault("total", total) 30 | with tqdm_class(**kwargs) as t: 31 | for i in itertools.product(*iterables): 32 | yield i 33 | t.update() 34 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/telegram.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends updates to a Telegram bot. 3 | 4 | Usage: 5 | >>> from tqdm.contrib.telegram import tqdm, trange 6 | >>> for i in trange(10, token='{token}', chat_id='{chat_id}'): 7 | ... ... 8 | 9 | ![screenshot]( 10 | https://raw.githubusercontent.com/tqdm/img/src/screenshot-telegram.gif) 11 | """ 12 | from __future__ import absolute_import 13 | from os import getenv 14 | 15 | from requests import Session 16 | 17 | from tqdm.auto import tqdm as tqdm_auto 18 | from tqdm.utils import _range 19 | from .utils_worker import MonoWorker 20 | __author__ = {"github.com/": ["casperdcl"]} 21 | __all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange'] 22 | 23 | 24 | class TelegramIO(MonoWorker): 25 | """Non-blocking file-like IO using a Telegram Bot.""" 26 | API = 'https://api.telegram.org/bot' 27 | 28 | def __init__(self, token, chat_id): 29 | """Creates a new message in the given `chat_id`.""" 30 | super(TelegramIO, self).__init__() 31 | self.token = token 32 | self.chat_id = chat_id 33 | self.session = session = Session() 34 | self.text = self.__class__.__name__ 35 | try: 36 | res = session.post( 37 | self.API + '%s/sendMessage' % self.token, 38 | data=dict(text='`' + self.text + '`', chat_id=self.chat_id, 39 | parse_mode='MarkdownV2')) 40 | except Exception as e: 41 | tqdm_auto.write(str(e)) 42 | else: 43 | self.message_id = res.json()['result']['message_id'] 44 | 45 | def write(self, s): 46 | """Replaces internal `message_id`'s text with `s`.""" 47 | if not s: 48 | return 49 | s = s.replace('\r', '').strip() 50 | if s == self.text: 51 | return # avoid duplicate message Bot error 52 | self.text = s 53 | try: 54 | future = self.submit( 55 | self.session.post, 56 | self.API + '%s/editMessageText' % self.token, 57 | data=dict( 58 | text='`' + s + '`', chat_id=self.chat_id, 59 | message_id=self.message_id, parse_mode='MarkdownV2')) 60 | except Exception as e: 61 | tqdm_auto.write(str(e)) 62 | else: 63 | return future 64 | 65 | 66 | class tqdm_telegram(tqdm_auto): 67 | """ 68 | Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot. 69 | May take a few seconds to create (`__init__`). 70 | 71 | - create a bot 72 | - copy its `{token}` 73 | - add the bot to a chat and send it a message such as `/start` 74 | - go to to find out 75 | the `{chat_id}` 76 | - paste the `{token}` & `{chat_id}` below 77 | 78 | >>> from tqdm.contrib.telegram import tqdm, trange 79 | >>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'): 80 | ... ... 81 | """ 82 | def __init__(self, *args, **kwargs): 83 | """ 84 | Parameters 85 | ---------- 86 | token : str, required. Telegram token 87 | [default: ${TQDM_TELEGRAM_TOKEN}]. 88 | chat_id : str, required. Telegram chat ID 89 | [default: ${TQDM_TELEGRAM_CHAT_ID}]. 90 | 91 | See `tqdm.auto.tqdm.__init__` for other parameters. 92 | """ 93 | kwargs = kwargs.copy() 94 | self.tgio = TelegramIO( 95 | kwargs.pop('token', getenv('TQDM_TELEGRAM_TOKEN')), 96 | kwargs.pop('chat_id', getenv('TQDM_TELEGRAM_CHAT_ID'))) 97 | super(tqdm_telegram, self).__init__(*args, **kwargs) 98 | 99 | def display(self, **kwargs): 100 | super(tqdm_telegram, self).display(**kwargs) 101 | fmt = self.format_dict 102 | if 'bar_format' in fmt and fmt['bar_format']: 103 | fmt['bar_format'] = fmt['bar_format'].replace('', '{bar}') 104 | else: 105 | fmt['bar_format'] = '{l_bar}{bar}{r_bar}' 106 | fmt['bar_format'] = fmt['bar_format'].replace('{bar}', '{bar:10u}') 107 | self.tgio.write(self.format_meter(**fmt)) 108 | 109 | def __new__(cls, *args, **kwargs): 110 | """ 111 | Workaround for mixed-class same-stream nested progressbars. 112 | See [#509](https://github.com/tqdm/tqdm/issues/509) 113 | """ 114 | with cls.get_lock(): 115 | try: 116 | cls._instances = tqdm_auto._instances 117 | except AttributeError: 118 | pass 119 | instance = super(tqdm_telegram, cls).__new__(cls, *args, **kwargs) 120 | with cls.get_lock(): 121 | try: 122 | # `tqdm_auto` may have been changed so update 123 | cls._instances.update(tqdm_auto._instances) 124 | except AttributeError: 125 | pass 126 | tqdm_auto._instances = cls._instances 127 | return instance 128 | 129 | 130 | def ttgrange(*args, **kwargs): 131 | """ 132 | A shortcut for `tqdm.contrib.telegram.tqdm(xrange(*args), **kwargs)`. 133 | On Python3+, `range` is used instead of `xrange`. 134 | """ 135 | return tqdm_telegram(_range(*args), **kwargs) 136 | 137 | 138 | # Aliases 139 | tqdm = tqdm_telegram 140 | trange = ttgrange 141 | -------------------------------------------------------------------------------- /bring2lite/tqdm/contrib/utils_worker.py: -------------------------------------------------------------------------------- 1 | """ 2 | IO/concurrency helpers for `tqdm.contrib`. 3 | """ 4 | from __future__ import absolute_import 5 | 6 | from concurrent.futures import ThreadPoolExecutor 7 | from collections import deque 8 | 9 | from tqdm.auto import tqdm as tqdm_auto 10 | __author__ = {"github.com/": ["casperdcl"]} 11 | __all__ = ['MonoWorker'] 12 | 13 | 14 | class MonoWorker(object): 15 | """ 16 | Supports one running task and one waiting task. 17 | The waiting task is the most recent submitted (others are discarded). 18 | """ 19 | def __init__(self): 20 | self.pool = ThreadPoolExecutor(max_workers=1) 21 | self.futures = deque([], 2) 22 | 23 | def submit(self, func, *args, **kwargs): 24 | """`func(*args, **kwargs)` may replace currently waiting task.""" 25 | futures = self.futures 26 | if len(futures) == futures.maxlen: 27 | running = futures.popleft() 28 | if not running.done(): 29 | if len(futures): # clear waiting 30 | waiting = futures.pop() 31 | waiting.cancel() 32 | futures.appendleft(running) # re-insert running 33 | try: 34 | waiting = self.pool.submit(func, *args, **kwargs) 35 | except Exception as e: 36 | tqdm_auto.write(str(e)) 37 | else: 38 | futures.append(waiting) 39 | return waiting 40 | -------------------------------------------------------------------------------- /bring2lite/tqdm/keras.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division 2 | from .auto import tqdm as tqdm_auto 3 | from copy import copy 4 | try: 5 | import keras 6 | except ImportError as e: 7 | try: 8 | from tensorflow import keras 9 | except ImportError: 10 | raise e 11 | __author__ = {"github.com/": ["casperdcl"]} 12 | __all__ = ['TqdmCallback'] 13 | 14 | 15 | class TqdmCallback(keras.callbacks.Callback): 16 | """`keras` callback for epoch and batch progress""" 17 | @staticmethod 18 | def bar2callback(bar, pop=None, delta=(lambda logs: 1)): 19 | def callback(_, logs=None): 20 | n = delta(logs) 21 | if logs: 22 | if pop: 23 | logs = copy(logs) 24 | [logs.pop(i, 0) for i in pop] 25 | bar.set_postfix(logs, refresh=False) 26 | bar.update(n) 27 | 28 | return callback 29 | 30 | def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, 31 | tqdm_class=tqdm_auto): 32 | """ 33 | Parameters 34 | ---------- 35 | epochs : int, optional 36 | data_size : int, optional 37 | Number of training pairs. 38 | batch_size : int, optional 39 | Number of training pairs per batch. 40 | verbose : int 41 | 0: epoch, 1: batch (transient), 2: batch. [default: 1]. 42 | Will be set to `0` unless both `data_size` and `batch_size` 43 | are given. 44 | tqdm_class : optional 45 | `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. 46 | """ 47 | self.tqdm_class = tqdm_class 48 | self.epoch_bar = tqdm_class(total=epochs, unit='epoch') 49 | self.on_epoch_end = self.bar2callback(self.epoch_bar) 50 | if data_size and batch_size: 51 | self.batches = batches = (data_size + batch_size - 1) // batch_size 52 | else: 53 | self.batches = batches = None 54 | self.verbose = verbose 55 | if verbose == 1: 56 | self.batch_bar = tqdm_class(total=batches, unit='batch', 57 | leave=False) 58 | self.on_batch_end = self.bar2callback( 59 | self.batch_bar, 60 | pop=['batch', 'size'], 61 | delta=lambda logs: logs.get('size', 1)) 62 | 63 | def on_train_begin(self, *_, **__): 64 | params = self.params.get 65 | auto_total = params('epochs', params('nb_epoch', None)) 66 | if auto_total is not None: 67 | self.epoch_bar.reset(total=auto_total) 68 | 69 | def on_epoch_begin(self, *_, **__): 70 | if self.verbose: 71 | params = self.params.get 72 | total = params('samples', params( 73 | 'nb_sample', params('steps', None))) or self.batches 74 | if self.verbose == 2: 75 | if hasattr(self, 'batch_bar'): 76 | self.batch_bar.close() 77 | self.batch_bar = self.tqdm_class( 78 | total=total, unit='batch', leave=True, 79 | unit_scale=1 / (params('batch_size', 1) or 1)) 80 | self.on_batch_end = self.bar2callback( 81 | self.batch_bar, 82 | pop=['batch', 'size'], 83 | delta=lambda logs: logs.get('size', 1)) 84 | elif self.verbose == 1: 85 | self.batch_bar.unit_scale = 1 / (params('batch_size', 1) or 1) 86 | self.batch_bar.reset(total=total) 87 | else: 88 | raise KeyError('Unknown verbosity') 89 | 90 | def on_train_end(self, *_, **__): 91 | if self.verbose: 92 | self.batch_bar.close() 93 | self.epoch_bar.close() 94 | 95 | @staticmethod 96 | def _implements_train_batch_hooks(): 97 | return True 98 | 99 | @staticmethod 100 | def _implements_test_batch_hooks(): 101 | return True 102 | 103 | @staticmethod 104 | def _implements_predict_batch_hooks(): 105 | return True 106 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/py37_asyncio.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from functools import partial, wraps 3 | from time import time 4 | 5 | from tests_tqdm import with_setup, pretest, posttest, StringIO, closing 6 | from tqdm.asyncio import tqdm_asyncio, tarange 7 | 8 | tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) 9 | trange = partial(tarange, miniters=0, mininterval=0) 10 | as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) 11 | 12 | 13 | def with_setup_sync(func): 14 | @with_setup(pretest, posttest) 15 | @wraps(func) 16 | def inner(): 17 | return asyncio.run(func()) 18 | return inner 19 | 20 | 21 | def count(start=0, step=1): 22 | i = start 23 | while True: 24 | new_start = yield i 25 | if new_start is None: 26 | i += step 27 | else: 28 | i = new_start 29 | 30 | 31 | async def acount(*args, **kwargs): 32 | for i in count(*args, **kwargs): 33 | yield i 34 | 35 | 36 | @with_setup_sync 37 | async def test_generators(): 38 | """Test asyncio generators""" 39 | with closing(StringIO()) as our_file: 40 | async for i in tqdm(count(), desc="counter", file=our_file): 41 | if i >= 8: 42 | break 43 | assert '9it' in our_file.getvalue() 44 | our_file.seek(0) 45 | our_file.truncate() 46 | 47 | async for i in tqdm(acount(), desc="async_counter", file=our_file): 48 | if i >= 8: 49 | break 50 | assert '9it' in our_file.getvalue() 51 | 52 | 53 | @with_setup_sync 54 | async def test_range(): 55 | """Test asyncio range""" 56 | with closing(StringIO()) as our_file: 57 | async for _ in tqdm(range(9), desc="range", file=our_file): 58 | pass 59 | assert '9/9' in our_file.getvalue() 60 | our_file.seek(0) 61 | our_file.truncate() 62 | 63 | async for _ in trange(9, desc="trange", file=our_file): 64 | pass 65 | assert '9/9' in our_file.getvalue() 66 | 67 | 68 | @with_setup_sync 69 | async def test_nested(): 70 | """Test asyncio nested""" 71 | with closing(StringIO()) as our_file: 72 | async for _ in tqdm(trange(9, desc="inner", file=our_file), 73 | desc="outer", file=our_file): 74 | pass 75 | assert 'inner: 100%' in our_file.getvalue() 76 | assert 'outer: 100%' in our_file.getvalue() 77 | 78 | 79 | @with_setup_sync 80 | async def test_coroutines(): 81 | """Test asyncio coroutine.send""" 82 | with closing(StringIO()) as our_file: 83 | with tqdm(count(), file=our_file) as pbar: 84 | async for i in pbar: 85 | if i == 9: 86 | pbar.send(-10) 87 | elif i < 0: 88 | assert i == -9 89 | break 90 | assert '10it' in our_file.getvalue() 91 | 92 | 93 | @with_setup_sync 94 | async def test_as_completed(): 95 | """Test asyncio as_completed""" 96 | with closing(StringIO()) as our_file: 97 | t = time() 98 | skew = time() - t 99 | for i in as_completed([asyncio.sleep(0.01 * i) 100 | for i in range(30, 0, -1)], file=our_file): 101 | await i 102 | assert 0.29 < time() - t - 2 * skew < 0.31 103 | assert '30/30' in our_file.getvalue() 104 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_asyncio.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info[:2] > (3, 6): 4 | from py37_asyncio import * # NOQA 5 | else: 6 | from tests_tqdm import SkipTest 7 | raise SkipTest 8 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_concurrent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib.concurrent`. 3 | """ 4 | from warnings import catch_warnings 5 | from tqdm.contrib.concurrent import thread_map, process_map 6 | from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ 7 | closing 8 | 9 | 10 | def incr(x): 11 | """Dummy function""" 12 | return x + 1 13 | 14 | 15 | @with_setup(pretest, posttest) 16 | def test_thread_map(): 17 | """Test contrib.concurrent.thread_map""" 18 | with closing(StringIO()) as our_file: 19 | a = range(9) 20 | b = [i + 1 for i in a] 21 | try: 22 | assert thread_map(lambda x: x + 1, a, file=our_file) == b 23 | except ImportError: 24 | raise SkipTest 25 | assert thread_map(incr, a, file=our_file) == b 26 | 27 | 28 | @with_setup(pretest, posttest) 29 | def test_process_map(): 30 | """Test contrib.concurrent.process_map""" 31 | with closing(StringIO()) as our_file: 32 | a = range(9) 33 | b = [i + 1 for i in a] 34 | try: 35 | assert process_map(incr, a, file=our_file) == b 36 | except ImportError: 37 | raise SkipTest 38 | 39 | 40 | def test_chunksize_warning(): 41 | """Test contrib.concurrent.process_map chunksize warnings""" 42 | try: 43 | from unittest.mock import patch 44 | except ImportError: 45 | raise SkipTest 46 | 47 | for iterables, should_warn in [ 48 | ([], False), 49 | (['x'], False), 50 | ([()], False), 51 | (['x', ()], False), 52 | (['x' * 1001], True), 53 | (['x' * 100, ('x',) * 1001], True), 54 | ]: 55 | with patch('tqdm.contrib.concurrent._executor_map'): 56 | with catch_warnings(record=True) as w: 57 | process_map(incr, *iterables) 58 | assert should_warn == bool(w) 59 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_contrib.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib`. 3 | """ 4 | import sys 5 | from tqdm.contrib import tenumerate, tzip, tmap 6 | from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ 7 | closing 8 | 9 | 10 | def incr(x): 11 | """Dummy function""" 12 | return x + 1 13 | 14 | 15 | @with_setup(pretest, posttest) 16 | def test_enumerate(): 17 | """Test contrib.tenumerate""" 18 | with closing(StringIO()) as our_file: 19 | a = range(9) 20 | assert list(tenumerate(a, file=our_file)) == list(enumerate(a)) 21 | assert list(tenumerate(a, 42, file=our_file)) == list(enumerate(a, 42)) 22 | with closing(StringIO()) as our_file: 23 | _ = list(tenumerate((i for i in a), file=our_file)) 24 | assert "100%" not in our_file.getvalue() 25 | with closing(StringIO()) as our_file: 26 | _ = list(tenumerate((i for i in a), file=our_file, total=len(a))) 27 | assert "100%" in our_file.getvalue() 28 | 29 | 30 | @with_setup(pretest, posttest) 31 | def test_enumerate_numpy(): 32 | """Test contrib.tenumerate(numpy.ndarray)""" 33 | try: 34 | import numpy as np 35 | except ImportError: 36 | raise SkipTest 37 | with closing(StringIO()) as our_file: 38 | a = np.random.random((42, 1337)) 39 | assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) 40 | 41 | 42 | @with_setup(pretest, posttest) 43 | def test_zip(): 44 | """Test contrib.tzip""" 45 | with closing(StringIO()) as our_file: 46 | a = range(9) 47 | b = [i + 1 for i in a] 48 | if sys.version_info[:1] < (3,): 49 | assert tzip(a, b, file=our_file) == zip(a, b) 50 | else: 51 | gen = tzip(a, b, file=our_file) 52 | assert gen != list(zip(a, b)) 53 | assert list(gen) == list(zip(a, b)) 54 | 55 | 56 | @with_setup(pretest, posttest) 57 | def test_map(): 58 | """Test contrib.tmap""" 59 | with closing(StringIO()) as our_file: 60 | a = range(9) 61 | b = [i + 1 for i in a] 62 | if sys.version_info[:1] < (3,): 63 | assert tmap(lambda x: x + 1, a, file=our_file) == map(incr, a) 64 | else: 65 | gen = tmap(lambda x: x + 1, a, file=our_file) 66 | assert gen != b 67 | assert list(gen) == b 68 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_itertools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib.itertools`. 3 | """ 4 | from tqdm.contrib.itertools import product 5 | from tests_tqdm import with_setup, pretest, posttest, StringIO, closing 6 | import itertools 7 | 8 | 9 | class NoLenIter(object): 10 | def __init__(self, iterable): 11 | self._it = iterable 12 | 13 | def __iter__(self): 14 | for i in self._it: 15 | yield i 16 | 17 | 18 | @with_setup(pretest, posttest) 19 | def test_product(): 20 | """Test contrib.itertools.product""" 21 | with closing(StringIO()) as our_file: 22 | a = range(9) 23 | assert list(product(a, a[::-1], file=our_file)) == \ 24 | list(itertools.product(a, a[::-1])) 25 | 26 | assert list(product(a, NoLenIter(a), file=our_file)) == \ 27 | list(itertools.product(a, NoLenIter(a))) 28 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_keras.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from tqdm import tqdm 3 | from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ 4 | closing 5 | 6 | 7 | @with_setup(pretest, posttest) 8 | def test_keras(): 9 | """Test tqdm.keras.TqdmCallback""" 10 | try: 11 | from tqdm.keras import TqdmCallback 12 | import numpy as np 13 | try: 14 | import keras as K 15 | except ImportError: 16 | from tensorflow import keras as K 17 | except ImportError: 18 | raise SkipTest 19 | 20 | # 1D autoencoder 21 | dtype = np.float32 22 | model = K.models.Sequential( 23 | [K.layers.InputLayer((1, 1), dtype=dtype), K.layers.Conv1D(1, 1)] 24 | ) 25 | model.compile("adam", "mse") 26 | x = np.random.rand(100, 1, 1).astype(dtype) 27 | batch_size = 10 28 | batches = len(x) / batch_size 29 | epochs = 5 30 | 31 | with closing(StringIO()) as our_file: 32 | 33 | class Tqdm(tqdm): 34 | """redirected I/O class""" 35 | 36 | def __init__(self, *a, **k): 37 | k.setdefault("file", our_file) 38 | super(Tqdm, self).__init__(*a, **k) 39 | 40 | # just epoch (no batch) progress 41 | model.fit( 42 | x, 43 | x, 44 | epochs=epochs, 45 | batch_size=batch_size, 46 | verbose=False, 47 | callbacks=[ 48 | TqdmCallback( 49 | epochs, 50 | data_size=len(x), 51 | batch_size=batch_size, 52 | verbose=0, 53 | tqdm_class=Tqdm, 54 | ) 55 | ], 56 | ) 57 | res = our_file.getvalue() 58 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 59 | assert "{batches}/{batches}".format(batches=batches) not in res 60 | 61 | # full (epoch and batch) progress 62 | our_file.seek(0) 63 | our_file.truncate() 64 | model.fit( 65 | x, 66 | x, 67 | epochs=epochs, 68 | batch_size=batch_size, 69 | verbose=False, 70 | callbacks=[ 71 | TqdmCallback( 72 | epochs, 73 | data_size=len(x), 74 | batch_size=batch_size, 75 | verbose=2, 76 | tqdm_class=Tqdm, 77 | ) 78 | ], 79 | ) 80 | res = our_file.getvalue() 81 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 82 | assert "{batches}/{batches}".format(batches=batches) in res 83 | 84 | # auto-detect epochs and batches 85 | our_file.seek(0) 86 | our_file.truncate() 87 | model.fit( 88 | x, 89 | x, 90 | epochs=epochs, 91 | batch_size=batch_size, 92 | verbose=False, 93 | callbacks=[TqdmCallback(verbose=2, tqdm_class=Tqdm)], 94 | ) 95 | res = our_file.getvalue() 96 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 97 | assert "{batches}/{batches}".format(batches=batches) in res 98 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import subprocess 3 | from os import path 4 | from shutil import rmtree 5 | from tempfile import mkdtemp 6 | from tqdm.cli import main, TqdmKeyError, TqdmTypeError 7 | from tqdm.utils import IS_WIN 8 | from io import open as io_open 9 | 10 | from tests_tqdm import with_setup, pretest, posttest, _range, closing, \ 11 | UnicodeIO, StringIO, SkipTest 12 | 13 | 14 | def _sh(*cmd, **kwargs): 15 | return subprocess.Popen(cmd, stdout=subprocess.PIPE, 16 | **kwargs).communicate()[0].decode('utf-8') 17 | 18 | 19 | class Null(object): 20 | def __call__(self, *_, **__): 21 | return self 22 | 23 | def __getattr__(self, _): 24 | return self 25 | 26 | 27 | IN_DATA_LIST = map(str, _range(int(123))) 28 | NULL = Null() 29 | 30 | 31 | # WARNING: this should be the last test as it messes with sys.stdin, argv 32 | @with_setup(pretest, posttest) 33 | def test_main(): 34 | """Test command line pipes""" 35 | ls_out = _sh('ls').replace('\r\n', '\n') 36 | ls = subprocess.Popen('ls', stdout=subprocess.PIPE, 37 | stderr=subprocess.STDOUT) 38 | res = _sh(sys.executable, '-c', 'from tqdm.cli import main; main()', 39 | stdin=ls.stdout, stderr=subprocess.STDOUT) 40 | ls.wait() 41 | 42 | # actual test: 43 | 44 | assert ls_out in res.replace('\r\n', '\n') 45 | 46 | # semi-fake test which gets coverage: 47 | _SYS = sys.stdin, sys.argv 48 | 49 | with closing(StringIO()) as sys.stdin: 50 | sys.argv = ['', '--desc', 'Test CLI --delim', 51 | '--ascii', 'True', '--delim', r'\0', '--buf_size', '64'] 52 | sys.stdin.write('\0'.join(map(str, _range(int(123))))) 53 | # sys.stdin.write(b'\xff') # TODO 54 | sys.stdin.seek(0) 55 | main() 56 | sys.stdin = IN_DATA_LIST 57 | 58 | sys.argv = ['', '--desc', 'Test CLI pipes', 59 | '--ascii', 'True', '--unit_scale', 'True'] 60 | import tqdm.__main__ # NOQA 61 | 62 | with closing(StringIO()) as sys.stdin: 63 | IN_DATA = '\0'.join(IN_DATA_LIST) 64 | sys.stdin.write(IN_DATA) 65 | sys.stdin.seek(0) 66 | sys.argv = ['', '--ascii', '--bytes=True', '--unit_scale', 'False'] 67 | with closing(UnicodeIO()) as fp: 68 | main(fp=fp) 69 | assert str(len(IN_DATA)) in fp.getvalue() 70 | sys.stdin = IN_DATA_LIST 71 | 72 | # test --log 73 | with closing(StringIO()) as sys.stdin: 74 | sys.stdin.write('\0'.join(map(str, _range(int(123))))) 75 | sys.stdin.seek(0) 76 | # with closing(UnicodeIO()) as fp: 77 | main(argv=['--log', 'DEBUG'], fp=NULL) 78 | # assert "DEBUG:" in sys.stdout.getvalue() 79 | sys.stdin = IN_DATA_LIST 80 | 81 | # clean up 82 | sys.stdin, sys.argv = _SYS 83 | 84 | 85 | def test_manpath(): 86 | """Test CLI --manpath""" 87 | if IS_WIN: 88 | raise SkipTest 89 | tmp = mkdtemp() 90 | man = path.join(tmp, "tqdm.1") 91 | assert not path.exists(man) 92 | try: 93 | main(argv=['--manpath', tmp], fp=NULL) 94 | except SystemExit: 95 | pass 96 | else: 97 | raise SystemExit("Expected system exit") 98 | assert path.exists(man) 99 | rmtree(tmp, True) 100 | 101 | 102 | def test_comppath(): 103 | """Test CLI --comppath""" 104 | if IS_WIN: 105 | raise SkipTest 106 | tmp = mkdtemp() 107 | man = path.join(tmp, "tqdm_completion.sh") 108 | assert not path.exists(man) 109 | try: 110 | main(argv=['--comppath', tmp], fp=NULL) 111 | except SystemExit: 112 | pass 113 | else: 114 | raise SystemExit("Expected system exit") 115 | assert path.exists(man) 116 | 117 | # check most important options appear 118 | with io_open(man, mode='r', encoding='utf-8') as fd: 119 | script = fd.read() 120 | opts = set([ 121 | '--help', '--desc', '--total', '--leave', '--ncols', '--ascii', 122 | '--dynamic_ncols', '--position', '--bytes', '--nrows', '--delim', 123 | '--manpath', '--comppath' 124 | ]) 125 | assert all(args in script for args in opts) 126 | rmtree(tmp, True) 127 | 128 | 129 | def test_exceptions(): 130 | """Test CLI Exceptions""" 131 | _SYS = sys.stdin, sys.argv 132 | sys.stdin = IN_DATA_LIST 133 | 134 | sys.argv = ['', '-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo'] 135 | try: 136 | main(fp=NULL) 137 | except TqdmKeyError as e: 138 | if 'bad_arg_u_ment' not in str(e): 139 | raise 140 | else: 141 | raise TqdmKeyError('bad_arg_u_ment') 142 | 143 | sys.argv = ['', '-ascii', '-unit_scale', 'invalid_bool_value'] 144 | try: 145 | main(fp=NULL) 146 | except TqdmTypeError as e: 147 | if 'invalid_bool_value' not in str(e): 148 | raise 149 | else: 150 | raise TqdmTypeError('invalid_bool_value') 151 | 152 | sys.argv = ['', '-ascii', '--total', 'invalid_int_value'] 153 | try: 154 | main(fp=NULL) 155 | except TqdmTypeError as e: 156 | if 'invalid_int_value' not in str(e): 157 | raise 158 | else: 159 | raise TqdmTypeError('invalid_int_value') 160 | 161 | # test SystemExits 162 | for i in ('-h', '--help', '-v', '--version'): 163 | sys.argv = ['', i] 164 | try: 165 | main(fp=NULL) 166 | except SystemExit: 167 | pass 168 | else: 169 | raise ValueError('expected SystemExit') 170 | 171 | # clean up 172 | sys.stdin, sys.argv = _SYS 173 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_notebook.py: -------------------------------------------------------------------------------- 1 | from tqdm.notebook import tqdm as tqdm_notebook 2 | from tests_tqdm import with_setup, pretest, posttest 3 | 4 | 5 | @with_setup(pretest, posttest) 6 | def test_notebook_disabled_description(): 7 | """Test that set_description works for disabled tqdm_notebook""" 8 | with tqdm_notebook(1, disable=True) as t: 9 | t.set_description("description") 10 | -------------------------------------------------------------------------------- /bring2lite/tqdm/tests/tests_version.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def test_version(): 5 | """Test version string""" 6 | from tqdm import __version__ 7 | version_parts = re.split('[.-]', __version__) 8 | assert 3 <= len(version_parts) # must have at least Major.minor.patch 9 | try: 10 | map(int, version_parts[:3]) 11 | except ValueError: 12 | raise TypeError('Version Major.minor.patch must be 3 integers') 13 | -------------------------------------------------------------------------------- /bs4/LICENSE: -------------------------------------------------------------------------------- 1 | Beautiful Soup is made available under the MIT license: 2 | 3 | Copyright (c) 2004-2017 Leonard Richardson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 20 | BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 21 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | 25 | Beautiful Soup incorporates code from the html5lib library, which is 26 | also made available under the MIT license. Copyright (c) 2006-2013 27 | James Graham and other contributors 28 | -------------------------------------------------------------------------------- /bs4/tests/__init__.py: -------------------------------------------------------------------------------- 1 | "The beautifulsoup tests." 2 | -------------------------------------------------------------------------------- /bs4/tests/test_docs.py: -------------------------------------------------------------------------------- 1 | "Test harness for doctests." 2 | 3 | # pylint: disable-msg=E0611,W0142 4 | 5 | __metaclass__ = type 6 | __all__ = [ 7 | 'additional_tests', 8 | ] 9 | 10 | import atexit 11 | import doctest 12 | import os 13 | #from pkg_resources import ( 14 | # resource_filename, resource_exists, resource_listdir, cleanup_resources) 15 | import unittest 16 | 17 | DOCTEST_FLAGS = ( 18 | doctest.ELLIPSIS | 19 | doctest.NORMALIZE_WHITESPACE | 20 | doctest.REPORT_NDIFF) 21 | 22 | 23 | # def additional_tests(): 24 | # "Run the doc tests (README.txt and docs/*, if any exist)" 25 | # doctest_files = [ 26 | # os.path.abspath(resource_filename('bs4', 'README.txt'))] 27 | # if resource_exists('bs4', 'docs'): 28 | # for name in resource_listdir('bs4', 'docs'): 29 | # if name.endswith('.txt'): 30 | # doctest_files.append( 31 | # os.path.abspath( 32 | # resource_filename('bs4', 'docs/%s' % name))) 33 | # kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS) 34 | # atexit.register(cleanup_resources) 35 | # return unittest.TestSuite(( 36 | # doctest.DocFileSuite(*doctest_files, **kwargs))) 37 | -------------------------------------------------------------------------------- /bs4/tests/test_html5lib.py: -------------------------------------------------------------------------------- 1 | """Tests to ensure that the html5lib tree builder generates good trees.""" 2 | 3 | import warnings 4 | 5 | try: 6 | from bs4.builder import HTML5TreeBuilder 7 | HTML5LIB_PRESENT = True 8 | except ImportError, e: 9 | HTML5LIB_PRESENT = False 10 | from bs4.element import SoupStrainer 11 | from bs4.testing import ( 12 | HTML5TreeBuilderSmokeTest, 13 | SoupTest, 14 | skipIf, 15 | ) 16 | 17 | @skipIf( 18 | not HTML5LIB_PRESENT, 19 | "html5lib seems not to be present, not testing its tree builder.") 20 | class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest): 21 | """See ``HTML5TreeBuilderSmokeTest``.""" 22 | 23 | @property 24 | def default_builder(self): 25 | return HTML5TreeBuilder() 26 | 27 | def test_soupstrainer(self): 28 | # The html5lib tree builder does not support SoupStrainers. 29 | strainer = SoupStrainer("b") 30 | markup = "

A bold statement.

" 31 | with warnings.catch_warnings(record=True) as w: 32 | soup = self.soup(markup, parse_only=strainer) 33 | self.assertEqual( 34 | soup.decode(), self.document_for(markup)) 35 | 36 | self.assertTrue( 37 | "the html5lib tree builder doesn't support parse_only" in 38 | str(w[0].message)) 39 | 40 | def test_correctly_nested_tables(self): 41 | """html5lib inserts tags where other parsers don't.""" 42 | markup = ('' 43 | '' 44 | "') 48 | 49 | self.assertSoupEquals( 50 | markup, 51 | '
Here's another table:" 45 | '' 46 | '' 47 | '
foo
Here\'s another table:' 52 | '
foo
' 53 | '
') 54 | 55 | self.assertSoupEquals( 56 | "" 57 | "" 58 | "
Foo
Bar
Baz
") 59 | 60 | def test_xml_declaration_followed_by_doctype(self): 61 | markup = ''' 62 | 63 | 64 | 65 | 66 | 67 |

foo

68 | 69 | ''' 70 | soup = self.soup(markup) 71 | # Verify that we can reach the

tag; this means the tree is connected. 72 | self.assertEqual(b"

foo

", soup.p.encode()) 73 | 74 | def test_reparented_markup(self): 75 | markup = '

foo

\n

bar

' 76 | soup = self.soup(markup) 77 | self.assertEqual(u"

foo

\n

bar

", soup.body.decode()) 78 | self.assertEqual(2, len(soup.find_all('p'))) 79 | 80 | 81 | def test_reparented_markup_ends_with_whitespace(self): 82 | markup = '

foo

\n

bar

\n' 83 | soup = self.soup(markup) 84 | self.assertEqual(u"

foo

\n

bar

\n", soup.body.decode()) 85 | self.assertEqual(2, len(soup.find_all('p'))) 86 | 87 | def test_reparented_markup_containing_identical_whitespace_nodes(self): 88 | """Verify that we keep the two whitespace nodes in this 89 | document distinct when reparenting the adjacent tags. 90 | """ 91 | markup = '
' 92 | soup = self.soup(markup) 93 | space1, space2 = soup.find_all(string=' ') 94 | tbody1, tbody2 = soup.find_all('tbody') 95 | assert space1.next_element is tbody1 96 | assert tbody2.next_element is space2 97 | 98 | def test_reparented_markup_containing_children(self): 99 | markup = '' 100 | soup = self.soup(markup) 101 | noscript = soup.noscript 102 | self.assertEqual("target", noscript.next_element) 103 | target = soup.find(string='target') 104 | 105 | # The 'aftermath' string was duplicated; we want the second one. 106 | final_aftermath = soup.find_all(string='aftermath')[-1] 107 | 108 | # The