├── systemtools ├── test │ ├── __init__.py │ ├── nohup.out │ ├── killbill-test.py │ ├── extract-gz-test.py │ ├── printertest.py │ ├── queuetest.py │ ├── sys-test.py │ ├── clearterminaltest.py │ ├── number.py │ ├── systemtest.py │ ├── duration.py │ ├── location.py │ ├── filetest.py │ ├── setpythonpath.py │ ├── output.log │ └── basics.py ├── config.py ├── __init__.py ├── jupyterutils.py ├── pythonpath.py ├── synchronized.py ├── enumeration.py ├── data │ ├── fr-names.txt │ └── fr-lastnames.txt ├── logger.py ├── printer.py ├── location.py ├── number.py ├── duration.py ├── system.py └── file.py ├── setup.py ├── setup.cfg ├── LICENCE.txt ├── .gitignore ├── .github └── workflows │ └── python-publish.yml └── README.md /systemtools/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /systemtools/config.py: -------------------------------------------------------------------------------- 1 | defaultTmpDir = None -------------------------------------------------------------------------------- /systemtools/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.3" 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /systemtools/test/nohup.out: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | OUTPUT_TYPE.nohup 3 | -------------------------------------------------------------------------------- /systemtools/test/killbill-test.py: -------------------------------------------------------------------------------- 1 | from systemtools.system import * 2 | 3 | 4 | bash("killbill infloop") -------------------------------------------------------------------------------- /systemtools/jupyterutils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | def getIpynbArgv(jupyterClue="ipykernel_launcher.py"): 4 | if len(sys.argv) == 0: 5 | return [] 6 | if jupyterClue in sys.argv[0]: 7 | return [] 8 | return sys.argv[2:] -------------------------------------------------------------------------------- /systemtools/pythonpath.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import os 5 | try: 6 | user_paths = os.environ['PYTHONPATH'].split(os.pathsep) 7 | except KeyError: 8 | user_paths = [] 9 | 10 | for current in user_paths: 11 | print(current) -------------------------------------------------------------------------------- /systemtools/test/extract-gz-test.py: -------------------------------------------------------------------------------- 1 | from systemtools.file import * 2 | from systemtools.location import * 3 | from systemtools.basics import * 4 | 5 | 6 | 7 | 8 | 9 | 10 | path = sortedGlob("/home/hayj/Downloads/*.bin")[0] 11 | 12 | bin2txtFile(path) 13 | 14 | 15 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = systools 3 | version = 0.1.3 4 | author = hayj 5 | description = A project gathering basic Python tools. 6 | url = https://github.com/hayj/SystemTools 7 | long_description = file: README.md 8 | long_description_content_type = text/markdown 9 | 10 | [options] 11 | packages = find: 12 | install_requires = 13 | psutil 14 | parsedatetime 15 | arrow 16 | pytz 17 | tzlocal 18 | sh 19 | pathlib; python_version < "3.4" 20 | tabulate 21 | requests 22 | xtract 23 | unidecode 24 | 25 | -------------------------------------------------------------------------------- /systemtools/test/printertest.py: -------------------------------------------------------------------------------- 1 | from systemtools.logger import * 2 | from systemtools.printer import * 3 | 4 | def test3(): 5 | class aaa: 6 | def __init__(self, logger=None, verbose=True): 7 | self.logger = logger 8 | self.verbose = verbose 9 | def bbb(self): 10 | bp([1, 2, 3], 3, self) 11 | bp([1, 2, 3], self, 3) 12 | bp([1, 2, 3], self.logger, 3, verbose=self.verbose) 13 | bp([1, 2, 3], 3, self.logger, verbose=self.verbose) 14 | logger = Logger() 15 | a = aaa(logger=logger, verbose=True) 16 | a.bbb() 17 | 18 | 19 | if __name__ == '__main__': 20 | test3() 21 | -------------------------------------------------------------------------------- /systemtools/test/queuetest.py: -------------------------------------------------------------------------------- 1 | from systemtools.basics import * 2 | from systemtools.duration import * 3 | import time 4 | from multiprocessing import Process 5 | 6 | def f(q): 7 | time.sleep(getRandomInt(10)) 8 | # time.sleep(1) 9 | q.put(None) 10 | 11 | if __name__ == '__main__': 12 | nbProcesses = 15 13 | pbar = ProgressBar(nbProcesses) 14 | q = pbar.startQueue() 15 | processes = [] 16 | for i in range(nbProcesses): 17 | p = Process(target=f, args=(q,)) 18 | processes.append(p) 19 | for p in processes: 20 | p.start() 21 | for p in processes: 22 | p.join() 23 | pbar.stopQueue() -------------------------------------------------------------------------------- /systemtools/test/sys-test.py: -------------------------------------------------------------------------------- 1 | # pew in st-venv python ~/Workspace/Python/Utils/SystemTools/systemtools/test/sys-test.py 2 | 3 | import re 4 | import socket 5 | 6 | 7 | from systemtools.system import * 8 | 9 | 10 | 11 | def test1(): 12 | def isHostname(hostname): 13 | return getHostname().startswith(hostname) 14 | def getHostname(): 15 | return socket.gethostname() 16 | def getTipiNumber(): 17 | try: 18 | return int(re.search("[0-9]+", getHostname()).group(0)) 19 | except: 20 | return None 21 | 22 | tipiNumber = getTipiNumber() 23 | if tipiNumber is None: 24 | print("Test...") 25 | elif tipiNumber < 8: 26 | print("Task 1...") 27 | else: 28 | print("Task 2...") 29 | 30 | 31 | def test2(): 32 | for i in range(1000): 33 | bash("killbill phantomjs") 34 | 35 | 36 | 37 | if __name__ == '__main__': 38 | test2() -------------------------------------------------------------------------------- /systemtools/test/clearterminaltest.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # print("aaaaaaaaaa bbbbbbbbbb") 4 | 5 | # # print(chr(27) + "[2J") 6 | 7 | import os 8 | import sys 9 | from enum import Enum 10 | import signal 11 | 12 | 13 | print(getOutputType()) 14 | exit() 15 | 16 | 17 | 18 | # import os 19 | # os.system('cls' if os.name == 'nt' else 'clear') 20 | 21 | size = os.get_terminal_size() 22 | 23 | print(size[0]) 24 | 25 | 26 | if signal.getsignal(signal.SIGHUP) == signal.SIG_DFL: # default action 27 | print("No SIGHUP handler") 28 | else: 29 | print("In nohup mode") 30 | 31 | 32 | import time 33 | for x in range (0,5): 34 | b = "Loading" + "." * x 35 | print (b, end="\r") 36 | time.sleep(1) 37 | 38 | 39 | 40 | import sys 41 | print("FAILED...") 42 | sys.stdout.write("\033[F") #back to previous line 43 | time.sleep(1) 44 | sys.stdout.write("\033[K") #clear line 45 | print("SUCCESS!") -------------------------------------------------------------------------------- /systemtools/test/number.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # pew in systemtools-venv python ./test/number.py 3 | 4 | import os 5 | import sys 6 | sys.path.append('../') 7 | 8 | import unittest 9 | import doctest 10 | from systemtools import number 11 | from systemtools.number import * 12 | 13 | # The level allow the unit test execution to choose only the top level test 14 | mini = 0 15 | maxi = 5 16 | assert mini <= maxi 17 | 18 | print("==============\nStarting unit tests...") 19 | 20 | if mini <= 0 <= maxi: 21 | class DocTest(unittest.TestCase): 22 | def testDoctests(self): 23 | """Run doctests""" 24 | doctest.testmod(number) 25 | 26 | if mini <= 1 <= maxi: 27 | class Test1(unittest.TestCase): 28 | def test1(self): 29 | pass 30 | 31 | 32 | 33 | if __name__ == '__main__': 34 | unittest.main() # Or execute as Python unit-test in eclipse 35 | 36 | 37 | print("Unit tests done.\n==============") -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 hayj 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dirs: 2 | *.egg-info/ 3 | build/ 4 | dist/ 5 | _trash/ 6 | trash/ 7 | 8 | *hayj.py 9 | 10 | # Misc files: 11 | *.sh 12 | !README.md 13 | TODO.md 14 | 15 | # Python 16 | *.pyc 17 | __pycache__/ 18 | 19 | # hj specific 20 | **/wm-dist/*.json 21 | 22 | # Eclipse 23 | .directory 24 | .project 25 | .metadata 26 | bin/ 27 | tmp/ 28 | *.tmp 29 | *.bak 30 | *.swp 31 | *~.nib 32 | local.properties 33 | .settings/ 34 | .loadpath 35 | .recommenders 36 | 37 | # External tool builders 38 | .externalToolBuilders/ 39 | 40 | # Locally stored "Eclipse launch configurations" 41 | *.launch 42 | 43 | # PyDev specific (Python IDE for Eclipse) 44 | *.pydevproject 45 | 46 | # CDT-specific (C/C++ Development Tooling) 47 | .cproject 48 | 49 | # Java annotation processor (APT) 50 | .factorypath 51 | 52 | # PDT-specific (PHP Development Tools) 53 | .buildpath 54 | 55 | # sbteclipse plugin 56 | .target 57 | 58 | # Tern plugin 59 | .tern-project 60 | 61 | # TeXlipse plugin 62 | .texlipse 63 | 64 | # STS (Spring Tool Suite) 65 | .springBeans 66 | 67 | # Code Recommenders 68 | .recommenders/ 69 | 70 | # Scala IDE specific (Scala & Java development for Eclipse) 71 | .cache-main 72 | .scala_dependencies 73 | .worksheet 74 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.x' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install build 33 | - name: Build package 34 | run: python -m build 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 37 | with: 38 | user: __token__ 39 | password: ${{ secrets.PYPI_API_TOKEN }} 40 | -------------------------------------------------------------------------------- /systemtools/test/systemtest.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # cd /home/hayj/Workspace/Python/Utils/SystemTools && python setup.py install && pew in systemtools-venv python ./systemtools/test/systemtest.py 3 | 4 | import os 5 | exec(compile(open(os.path.dirname(os.path.abspath(__file__)) + "/setpythonpath.py").read(), os.path.dirname(os.path.abspath(__file__)) + "/setpythonpath.py", 'exec'), {}) 6 | 7 | 8 | import unittest 9 | import doctest 10 | from systemtools import system 11 | from systemtools.system import * 12 | import sh 13 | import glob 14 | 15 | # The level allow the unit test execution to choose only the top level test 16 | min = 1 17 | max = 1 18 | assert min <= max 19 | 20 | print("==============\nStarting unit tests...") 21 | 22 | if min <= 0 <= max: 23 | class DocTest(unittest.TestCase): 24 | def testDoctests(self): 25 | """Run doctests""" 26 | doctest.testmod(system) 27 | 28 | if min <= 1 <= max: 29 | class Test1(unittest.TestCase): 30 | def test1(self): 31 | 32 | argvOptionsToDict(argv=["thecommand", "abcd.md", "-r", "rrr", "afile.txt"]) 33 | argvOptionsToDict(argv=["thecommand", "abcd.md", "-r"]) 34 | 35 | if __name__ == '__main__': 36 | unittest.main() # Or execute as Python unit-test in eclipse 37 | 38 | 39 | print("Unit tests done.\n==============") 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /systemtools/test/duration.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | 5 | 6 | import unittest 7 | import doctest 8 | from systemtools import duration 9 | from systemtools.duration import *; 10 | import time 11 | 12 | # The level allow the unit test execution to choose only the top level test 13 | min = 0 14 | max = 1 15 | assert min <= max 16 | 17 | print("\n==============\nStarting unit tests...") 18 | 19 | if min <= 0 <= max: 20 | class DocTest(unittest.TestCase): 21 | def testDoctests(self): 22 | """Run doctests""" 23 | doctest.testmod(duration) 24 | 25 | if min <= 1 <= max: 26 | class Test1(unittest.TestCase): 27 | def testTicToc(self): 28 | tt = TicToc(maxDecimal=4); 29 | tt.tic(); 30 | time.sleep(1); 31 | temp = tt.tic(); 32 | self.assertTrue(temp > 0.9) 33 | time.sleep(1); 34 | tt.tic(); 35 | time.sleep(1); 36 | tt.setMaxDecimal(2); 37 | tt.toc(); 38 | time.sleep(1); 39 | tt.tic(); 40 | time.sleep(1); 41 | end = tt.toc(); 42 | self.assertTrue(end > 4.9) 43 | 44 | if __name__ == '__main__': 45 | unittest.main() # Or execute as Python unit-test in eclipse 46 | 47 | 48 | print("Unit tests done.\n==============") 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /systemtools/test/location.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # cd /home/hayj/Workspace/Python/Utils/SystemTools && python setup.py install && pew in systemtools-venv python ./systemtools/test/systemtest.py 3 | 4 | import os 5 | # exec(compile(open(os.path.dirname(os.path.abspath(__file__)) + "/setpythonpath.py").read(), os.path.dirname(os.path.abspath(__file__)) + "/setpythonpath.py", 'exec'), {}) 6 | 7 | 8 | import unittest 9 | import doctest 10 | from systemtools import location 11 | from systemtools.location import * 12 | 13 | # The level allow the unit test execution to choose only the top level test 14 | min = 0 15 | max = 1 16 | assert min <= max 17 | 18 | print("==============\nStarting unit tests...") 19 | 20 | if min <= 0 <= max: 21 | class DocTest(unittest.TestCase): 22 | def testDoctests(self): 23 | """Run doctests""" 24 | doctest.testmod(location) 25 | 26 | if min <= 1 <= max: 27 | class Test1(unittest.TestCase): 28 | def test1(self): 29 | self.assertTrue(tmpDir() == "/home/hayj/tmp") 30 | self.assertTrue(tmpDir("aaa") == "/home/hayj/tmp/aaa") 31 | from systemtools import config as systConf 32 | systConf.tmpDirPath = "/home/hayj/tmp/tmp" 33 | self.assertTrue(tmpDir() == "/home/hayj/tmp/tmp") 34 | self.assertTrue(tmpDir("aaa") == "/home/hayj/tmp/tmp/aaa") 35 | 36 | 37 | if __name__ == '__main__': 38 | unittest.main() # Or execute as Python unit-test in eclipse 39 | 40 | 41 | print("Unit tests done.\n==============") 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /systemtools/test/filetest.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # pew in systemtools-venv python ./test/filetest.py 3 | 4 | import os 5 | import sys 6 | from pip.utils.deprecation import RemovedInPip10Warning 7 | sys.path.append('../') 8 | 9 | import unittest 10 | import doctest 11 | from systemtools.basics import * 12 | from systemtools import file 13 | from systemtools.file import * 14 | 15 | # The level allow the unit test execution to choose only the top level test 16 | min = 0 17 | max = 10 18 | assert min <= max 19 | 20 | print("==============\nStarting unit tests...") 21 | 22 | if min <= 0 <= max: 23 | class DocTest(unittest.TestCase): 24 | def testDoctests(self): 25 | """Run doctests""" 26 | doctest.testmod(file) 27 | 28 | if min <= 1 <= max: 29 | class Test1(unittest.TestCase): 30 | def test1(self): 31 | print("Please check by hand:") 32 | input() 33 | file1 = "/home/hayj/Data/Misc/crawling/shorteners.txt" 34 | file2 = "/home/hayj/Data/Misc/crawling/proxies/proxies-failed.txt" 35 | file3 = "/home/hayj/Data/Misc/crawling/proxies/proxies-renew.txt" 36 | 37 | printLTS(fileToStrList(file1)) 38 | printLTS(fileToStrList(file1, removeDuplicates=True)) 39 | input() 40 | printLTS(fileToStrList(file2)) 41 | printLTS(fileToStrList(file2, commentStart="==>")) 42 | input() 43 | printLTS(fileToStrList(file3)) 44 | printLTS(fileToStrList(file3, skipBlank=False)) 45 | 46 | if __name__ == '__main__': 47 | unittest.main() # Or execute as Python unit-test in eclipse 48 | 49 | 50 | print("Unit tests done.\n==============") -------------------------------------------------------------------------------- /systemtools/synchronized.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import thread 4 | import threading 5 | import types 6 | 7 | def synchronized_with_attr(lock_name): 8 | 9 | def decorator(method): 10 | 11 | def synced_method(self, *args, **kws): 12 | lock = getattr(self, lock_name) 13 | with lock: 14 | return method(self, *args, **kws) 15 | 16 | return synced_method 17 | 18 | return decorator 19 | 20 | 21 | def syncronized_with(lock): 22 | 23 | def synchronized_obj(obj): 24 | 25 | if type(obj) is types.FunctionType: 26 | 27 | obj.__lock__ = lock 28 | 29 | def func(*args, **kws): 30 | with lock: 31 | obj(*args, **kws) 32 | return func 33 | 34 | elif type(obj) is types.ClassType: 35 | 36 | orig_init = obj.__init__ 37 | def __init__(self, *args, **kws): 38 | self.__lock__ = lock 39 | orig_init(self, *args, **kws) 40 | obj.__init__ = __init__ 41 | 42 | for key in obj.__dict__: 43 | val = obj.__dict__[key] 44 | if type(val) is types.FunctionType: 45 | decorator = syncronized_with(lock) 46 | obj.__dict__[key] = decorator(val) 47 | 48 | return obj 49 | 50 | return synchronized_obj 51 | 52 | 53 | def synchronized(item): 54 | 55 | if type(item) is types.StringType: 56 | decorator = synchronized_with_attr(item) 57 | return decorator(item) 58 | 59 | if type(item) is thread.LockType: 60 | decorator = syncronized_with(item) 61 | return decorator(item) 62 | 63 | else: 64 | new_lock = threading.Lock() 65 | decorator = syncronized_with(new_lock) 66 | return decorator(item) 67 | 68 | 69 | if __name__ == '__main__': 70 | pass -------------------------------------------------------------------------------- /systemtools/enumeration.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from enum import Enum 3 | 4 | 5 | 6 | def e(a, b=None, raiseException=False, logger=None): 7 | errorText1 = "Please give at least an enum.EnumMeta or an Enum" 8 | errorText2 = "Please give at least an enum.EnumMeta" 9 | if a is None: 10 | raise Exception(errorText1) 11 | if b is None: 12 | if isinstance(a, enum.EnumMeta): 13 | return [x.name for x in a] 14 | elif isinstance(a, Enum): 15 | return a.name 16 | else: 17 | if raiseException: 18 | raise Exception(errorText) 19 | else: 20 | if logger is not None: 21 | logger.error(errorText1) 22 | return a 23 | else: 24 | theEnumMeta = None 25 | theValue = None 26 | if isinstance(a, enum.EnumMeta): 27 | theEnumMeta = a 28 | theValue = b 29 | elif isinstance(b, enum.EnumMeta): 30 | theEnumMeta = b 31 | theValue = a 32 | else: 33 | if raiseException: 34 | raise Exception(errorText2) 35 | else: 36 | if logger is not None: 37 | logger.error(errorText2) 38 | return theValue 39 | theValue = enumCast(theValue, theEnumMeta, raiseException=raiseException) 40 | if not isinstance(theValue, Enum): 41 | errorText3 = "Cannot convert " + str(theValue) + " to an Enum" 42 | if raiseException: 43 | raise Exception(errorText3) 44 | else: 45 | if logger is not None: 46 | logger.error(errorText3) 47 | return theValue 48 | return theValue 49 | 50 | 51 | def enumCast(text, theEnum, logger=None, verbose=True, raiseException=False): 52 | try: 53 | if isinstance(text, Enum): 54 | return text 55 | elif isinstance(text, int): 56 | return list(theEnum)[text - 1] 57 | else: 58 | return theEnum[text] 59 | except Exception as e: 60 | errorText1 = "Cannot find " + str(text) + " in " + str(theEnum) 61 | if raiseException: 62 | raise Exception(errorText1) 63 | else: 64 | if logger is not None: 65 | logger.error(errorText1) 66 | return None 67 | 68 | def enumEquals(a, b): 69 | if isinstance(a, str) or isinstance(b, str): 70 | if isinstance(a, Enum): 71 | a = a.name 72 | if isinstance(b, Enum): 73 | b = b.name 74 | return a == b 75 | 76 | 77 | 78 | 79 | 80 | if __name__ == '__main__': 81 | DATASET = Enum("DATASET", "year2015 year2016") 82 | 83 | 84 | print(e(DATASET)) 85 | print() 86 | print(e(DATASET.year2015)) 87 | print(e(DATASET.year2016)) 88 | print() 89 | print(e("year2015", DATASET)) 90 | print(e("year2016", DATASET)) 91 | print() 92 | print(e(DATASET.year2015, DATASET)) 93 | print(e(DATASET.year2016, DATASET)) 94 | print() 95 | print(e(DATASET.year2015.value, DATASET)) 96 | print(e(DATASET.year2016.value, DATASET)) 97 | -------------------------------------------------------------------------------- /systemtools/data/fr-names.txt: -------------------------------------------------------------------------------- 1 | Jean 2 | Philippe 3 | Michel 4 | Alain 5 | Patrick 6 | Nicolas 7 | Christophe 8 | Pierre 9 | Christian 10 | Éric 11 | Frédéric 12 | Laurent 13 | Stéphane 14 | David 15 | Pascal 16 | Daniel 17 | Alexandre 18 | Julien 19 | Thierry 20 | Olivier 21 | Bernard 22 | Thomas 23 | Sébastien 24 | Gérard 25 | Didier 26 | Dominique 27 | Vincent 28 | François 29 | Bruno 30 | Guillaume 31 | Jérôme 32 | Jacques 33 | Marc 34 | Maxime 35 | Romain 36 | Claude 37 | Antoine 38 | Franck 39 | Jean-Pierre 40 | Anthony 41 | Kévin 42 | Gilles 43 | Cédric 44 | Serge 45 | André 46 | Mathieu 47 | Benjamin 48 | Patrice 49 | Fabrice 50 | Joël 51 | Jérémy 52 | Clément 53 | Arnaud 54 | Denis 55 | Paul 56 | Lucas 57 | Hervé 58 | Jean-Claude 59 | Sylvain 60 | Yves 61 | Ludovic 62 | Guy 63 | Florian 64 | Damien 65 | Alexis 66 | Mickaël 67 | Quentin 68 | Emmanuel 69 | Louis 70 | Benoît 71 | Jean-Luc 72 | Fabien 73 | Francis 74 | Hugo 75 | Jonathan 76 | Loïc 77 | Xavier 78 | Théo 79 | Adrien 80 | Raphaël 81 | Jean-François 82 | Grégory 83 | Robert 84 | Michaël 85 | Valentin 86 | Cyril 87 | Jean-Marc 88 | René 89 | Lionel 90 | Yannick 91 | Enzo 92 | Yannis 93 | Jean-Michel 94 | Baptiste 95 | Matthieu 96 | Rémi 97 | Georges 98 | Aurélien 99 | Nathan 100 | Jean-Paul 101 | Marie 102 | Nathalie 103 | Isabelle 104 | Sylvie 105 | Catherine 106 | Martine 107 | Christine 108 | Françoise 109 | Valérie 110 | Sandrine 111 | Stéphanie 112 | Véronique 113 | Sophie 114 | Céline 115 | Chantal 116 | Patricia 117 | Anne 118 | Brigitte 119 | Julie 120 | Monique 121 | Aurélie 122 | Nicole 123 | Laurence 124 | Annie 125 | Émilie 126 | Dominique 127 | Virginie 128 | Corinne 129 | Élodie 130 | Christelle 131 | Camille 132 | Caroline 133 | Léa 134 | Sarah 135 | Florence 136 | Laetitia 137 | Audrey 138 | Hélène 139 | Laura 140 | Manon 141 | Michèle 142 | Cécile 143 | Christiane 144 | Béatrice 145 | Claire 146 | Nadine 147 | Delphine 148 | Pauline 149 | Karine 150 | Mélanie 151 | Marion 152 | Chloe 153 | Jacqueline 154 | Elisabeth 155 | Evelyne 156 | Marine 157 | Claudine 158 | Anais 159 | Lucie 160 | Danielle 161 | Carole 162 | Fabienne 163 | Mathilde 164 | Sandra 165 | Pascale 166 | Annick 167 | Charlotte 168 | Emma 169 | Severine 170 | Sabrina 171 | Amandine 172 | Myriam 173 | Jocelyne 174 | Alexandra 175 | Angelique 176 | Josiane 177 | Joelle 178 | Agnes 179 | Mireille 180 | Vanessa 181 | Justine 182 | Sonia 183 | Bernadette 184 | Emmanuelle 185 | Oceane 186 | Amelie 187 | Clara 188 | Maryse 189 | Anne-marie 190 | Fanny 191 | Magali 192 | Marie-christine 193 | Morgane 194 | Ines 195 | Nadia 196 | Muriel 197 | Jessica 198 | Laure 199 | Genevieve 200 | Estelle -------------------------------------------------------------------------------- /systemtools/test/setpythonpath.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | ########## WORSPACE IN PYTHON PATH ########## 5 | # Include all Paths : 6 | # TODO faire un include de tous les sous dossiers du Workspace en remontant 4 dossiers 7 | # TODO (mettre une variable pour le nombre de dossier a remonter). 8 | # TODO mettre aussi le changement en utf-8 9 | # TODO Et mettre le os.chdir("??") aussi 10 | # TODO Faire une sorte d'entete pour chaque fichier python 11 | # Updated 09/03/17 12 | workspaceName = "Python" # Set the name of the Workspace in all ancestors folders 13 | onlyInit = False # TODO 14 | import sys 15 | import os 16 | import re 17 | def dirsHasRegex(dirs, regex): 18 | if not isinstance(dirs, list): 19 | dirs = [dirs] 20 | # for all subdirs: 21 | for currentDir in dirs: 22 | # We get all files: 23 | files = getFileNames(currentDir) 24 | # We check if there .py or __init__.py: 25 | for fname in files: 26 | if re.search(regex, fname) is not None: 27 | return True 28 | return False 29 | 30 | def getFileNames(currentFolder): 31 | files = [item for item in os.listdir(currentFolder) if os.path.isfile(os.path.join(currentFolder, item))] 32 | return files 33 | 34 | def getSubdirsPaths(currentFolder): 35 | dirs = [currentFolder + item + "/" for item in os.listdir(currentFolder) if os.path.isdir(os.path.join(currentFolder, item))] 36 | return dirs 37 | 38 | # We get the Workspace dir: 39 | currentFolder = os.path.dirname(os.path.abspath("__file__")) # If the script is executed in an other script 40 | # currentFolder = os.path.dirname(os.path.abspath(__file__)) # If this is pasted in the main script 41 | workspace = currentFolder.split(workspaceName)[0] + workspaceName + "/" 42 | # And while there are folders to watch: 43 | penddingFolderList = [workspace] 44 | pathsToAdd = [] 45 | while len(penddingFolderList) != 0: 46 | # We pop the current folder in in the pending queue: 47 | currentFolder = penddingFolderList.pop(0) 48 | if not currentFolder.split("/")[-2].startswith("."): 49 | hasPy = dirsHasRegex(currentFolder, ".py$") 50 | atLeastOneSubDirHasInit = dirsHasRegex(getSubdirsPaths(currentFolder), "__init__.py") 51 | # We add it in the path list: 52 | if hasPy or atLeastOneSubDirHasInit: 53 | pathsToAdd.append(currentFolder) 54 | # And We add the subfolders in the pending queue if there is no __init__.py: 55 | if not atLeastOneSubDirHasInit: 56 | subDirs = getSubdirsPaths(currentFolder) 57 | penddingFolderList += subDirs 58 | # We add all paths in the python path: 59 | for path in pathsToAdd: 60 | sys.path.append(path) 61 | # print path 62 | ############################################# -------------------------------------------------------------------------------- /systemtools/test/output.log: -------------------------------------------------------------------------------- 1 | 2019-08-26 14:03:59,924 :: INFO :: [ 1, 2, 3 ] 2 | 2019-08-26 14:03:59,925 :: ERROR :: Exception location: bp 3 | Exception type: 4 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 5 | Traceback (most recent call last): 6 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 61, in bp 7 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 8 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 9 | 10 | 2019-08-26 14:06:09,432 :: INFO :: [ 1, 2, 3 ] 11 | 2019-08-26 14:15:36,345 :: INFO :: [ 1, 2, 3 ] 12 | 2019-08-26 14:15:50,809 :: INFO :: [ 1, 2, 3 ] 13 | 2019-08-26 14:15:53,665 :: INFO :: [ 1, 2, 3 ] 14 | 2019-08-26 14:15:53,666 :: INFO :: [ 1, 2, 3 ] 15 | 2019-08-26 14:15:53,666 :: INFO :: [ 1, 2, 3 ] 16 | 2019-08-26 14:15:53,666 :: INFO :: [ 1, 2, 3 ] 17 | 2019-08-26 14:16:07,926 :: INFO :: [ 1, 2, 3 ] 18 | 2019-08-26 14:16:07,926 :: INFO :: [ 1, 2, 3 ] 19 | 2019-08-26 14:16:07,927 :: INFO :: [ 1, 2, 3 ] 20 | 2019-08-26 14:16:07,927 :: INFO :: [ 1, 2, 3 ] 21 | 2019-08-26 14:16:18,635 :: ERROR :: Exception location: bp 22 | Exception type: 23 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 24 | Traceback (most recent call last): 25 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 26 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 27 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 28 | 29 | 2019-08-26 14:16:18,636 :: ERROR :: Exception location: bp 30 | Exception type: 31 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 32 | Traceback (most recent call last): 33 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 34 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 35 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 36 | 37 | 2019-08-26 14:16:18,637 :: ERROR :: Exception location: bp 38 | Exception type: 39 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 40 | Traceback (most recent call last): 41 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 42 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 43 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 44 | 45 | 2019-08-26 14:16:18,637 :: ERROR :: Exception location: bp 46 | Exception type: 47 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 48 | Traceback (most recent call last): 49 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 50 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 51 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 52 | 53 | 2019-08-26 14:16:48,388 :: ERROR :: Exception location: bp 54 | Exception type: 55 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 56 | Traceback (most recent call last): 57 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 58 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 59 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 60 | 61 | 2019-08-26 14:16:48,389 :: ERROR :: Exception location: bp 62 | Exception type: 63 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 64 | Traceback (most recent call last): 65 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 66 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 67 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 68 | 69 | 2019-08-26 14:16:48,389 :: ERROR :: Exception location: bp 70 | Exception type: 71 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 72 | Traceback (most recent call last): 73 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 74 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 75 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 76 | 77 | 2019-08-26 14:16:48,389 :: ERROR :: Exception location: bp 78 | Exception type: 79 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 80 | Traceback (most recent call last): 81 | File "/home/hayj/Workspace/Python/Utils/SystemTools/systemtools/printer.py", line 65, in bp 82 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 83 | Exception: You must give level as an int or str and logger as a logger or as an object which has logger in its attributes 84 | 85 | 2019-08-26 14:16:53,883 :: INFO :: [ 1, 2, 3 ] 86 | 2019-08-26 14:16:53,884 :: INFO :: [ 1, 2, 3 ] 87 | 2019-08-26 14:16:53,884 :: INFO :: [ 1, 2, 3 ] 88 | 2019-08-26 14:16:53,884 :: INFO :: [ 1, 2, 3 ] 89 | -------------------------------------------------------------------------------- /systemtools/logger.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # http://sametmax.com/ecrire-des-logs-en-python/ 3 | 4 | # pew in st-venv python ~/Workspace/Python/Utils/SystemTools/systemtools/logger.py 5 | 6 | import logging 7 | from logging.handlers import RotatingFileHandler 8 | from systemtools.location import * 9 | from systemtools.system import * 10 | from systemtools.basics import * 11 | from systemtools.file import * 12 | from enum import Enum 13 | import traceback 14 | import smtplib 15 | import unidecode 16 | 17 | 18 | LOGTYPE = Enum('LOGTYPE', 'error info warning') 19 | 20 | 21 | def logException(e=None, obj=None, message=None, location=None, logger=None, verbose=True): 22 | if message is None: 23 | message = "" 24 | else: 25 | message += "\n" 26 | if location is not None: 27 | message += "Exception location: " + location + "\n" 28 | if e is not None: 29 | message += "Exception type: " + str(type(e)) + "\n" 30 | message += "Exception: " + str(e) 31 | try: 32 | message += "\n" + traceback.format_exc() 33 | except: pass 34 | log(message, obj=obj, logger=logger, verbose=verbose, logtype=LOGTYPE.error) 35 | def logInfo(text, obj=None, logger=None, verbose=True): 36 | log(text, obj=obj, logger=logger, verbose=verbose, logtype=LOGTYPE.info) 37 | def logWarning(text, obj=None, logger=None, verbose=True): 38 | log(text, obj=obj, logger=logger, verbose=verbose, logtype=LOGTYPE.warning) 39 | def logError(text, obj=None, logger=None, verbose=True): 40 | log(text, obj=obj, logger=logger, verbose=verbose, logtype=LOGTYPE.error) 41 | 42 | 43 | def log(text, obj=None, logger=None, verbose=True, logtype=LOGTYPE.info): 44 | """ 45 | This obj must have a logger and a verbose attr. 46 | Else you can give it in params. 47 | """ 48 | if obj is not None and isinstance(obj, Logger): 49 | logger, obj = obj, logger 50 | if logger is not None and not isinstance(logger, Logger): 51 | logger, obj = obj, logger 52 | if obj is not None: 53 | try: 54 | obj.logger 55 | obj.verbose 56 | except NameError: 57 | print(text) 58 | else: 59 | if obj.verbose: 60 | if obj.logger is None: 61 | print(text) 62 | else: 63 | logWithLogger(text, obj.logger, logtype) 64 | elif verbose: 65 | if logger is None: 66 | print(text) 67 | else: 68 | logWithLogger(text, logger, logtype) 69 | 70 | def logWithLogger(text, logger, logtype): 71 | if logtype == LOGTYPE.info: 72 | logger.info(text) 73 | if logtype == LOGTYPE.error: 74 | logger.error(text) 75 | if logtype == LOGTYPE.warning: 76 | logger.warning(text) 77 | 78 | 79 | class Logger(): 80 | def __init__(self, outputPath=None, moWeightMax=1, prefix=None, remove=False, doPrint=True): 81 | self.prefix = prefix 82 | if self.prefix is None: 83 | self.prefix = "" 84 | self.outputPath = outputPath 85 | if self.outputPath is None: 86 | self.outputPath = tmpDir() + "/noname.log" 87 | self.moWeightMax = moWeightMax 88 | self.randomName = getRandomStr() 89 | # Now we remove the previous log file: 90 | if remove: 91 | removeIfExists(self.outputPath) 92 | # création de l'objet logger qui va nous servir à écrire dans les logs 93 | logger = logging.getLogger(self.randomName) 94 | # on met le niveau du logger à DEBUG, comme ça il écrit tout 95 | logger.setLevel(logging.DEBUG) 96 | # création d'un formateur qui va ajouter le temps, le niveau 97 | # de chaque message quand on écrira un message dans le log 98 | formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s') 99 | # création d'un handler qui va rediriger une écriture du log vers 100 | # un fichier en mode 'append', avec 1 backup et une taille max de 1Mo 101 | file_handler = RotatingFileHandler(self.outputPath, 'a', self.moWeightMax * 1000000, 1) 102 | # on lui met le niveau sur DEBUG, on lui dit qu'il doit utiliser le formateur 103 | # créé précédement et on ajoute ce handler au logger 104 | file_handler.setLevel(logging.DEBUG) 105 | file_handler.setFormatter(formatter) 106 | logger.addHandler(file_handler) 107 | # création d'un second handler qui va rediriger chaque écriture de log 108 | # sur la console 109 | if doPrint: 110 | stream_handler = logging.StreamHandler() 111 | stream_handler.setLevel(logging.DEBUG) 112 | logger.addHandler(stream_handler) 113 | # We store the logger: 114 | self.logger = logger 115 | 116 | def remove(self, minSlashCount=3): 117 | if isFile(self.outputPath): 118 | remove(self.outputPath, minSlashCount=minSlashCount) 119 | if isFile(self.outputPath + ".1"): 120 | remove(self.outputPath, minSlashCount=minSlashCount) 121 | 122 | def prefixText(self, text): 123 | return self.prefix + str(text) 124 | 125 | def info(self, text): 126 | self.logger.info(self.prefixText(text)) 127 | def p(self, text): 128 | self.info(text) 129 | def log(self, text): 130 | self.info(text) 131 | def i(self, text): 132 | self.info(text) 133 | def warning(self, text): 134 | self.logger.warning(self.prefixText(text)) 135 | def w(self, text): 136 | self.warning(text) 137 | def e(self, text): 138 | self.error(text) 139 | def error(self, text): 140 | self.logger.error(self.prefixText(text)) 141 | 142 | def __repr__(self): 143 | return str("Logger " + str(self.outputPath)) 144 | 145 | def getLogger(self): 146 | return self.logger 147 | 148 | 149 | 150 | def notif(subject, content="", to=None, logger=None, verbose=True, name='HJWeb Watcher', sender='hjwebwatcher@gmail.com', password=None, doPrint=False, test=False, doStripAccents=True): 151 | if not isinstance(subject, str): 152 | subject = lts(subject) 153 | if not isinstance(content, str): 154 | content = lts(content) 155 | if subject is not None: 156 | subject = subject.strip() 157 | if subject is not None and content is None: 158 | if "\n" in subject: 159 | content = subject 160 | subject = None 161 | try: 162 | if to is None: 163 | to = sender 164 | toName = name 165 | else: 166 | toName = "You" 167 | if password is None: 168 | try: 169 | from datatools.dataencryptor import DataEncryptor 170 | password = DataEncryptor()["gmailauth"][sender] 171 | except Exception as e: 172 | logException(e, logger, verbose=verbose) 173 | newline = "\n" 174 | email = \ 175 | """From: %s <%s> 176 | To: %s <%s> 177 | Subject: %s 178 | 179 | %s 180 | """ % (name, sender, toName, to, subject, content) 181 | if doStripAccents: 182 | email = unidecode.unidecode(email) 183 | if doPrint and not test: 184 | log(subject + "\n" + content, logger, verbose=verbose) 185 | if test: 186 | log("\n" + subject + "\n" + content, logger, verbose=verbose) 187 | else: 188 | server = smtplib.SMTP_SSL('smtp.gmail.com', 465) 189 | server.ehlo() 190 | # server.starttls() # not working 191 | server.login(sender, password) 192 | server.sendmail(sender, to, email) 193 | server.close() 194 | # server.quit() # not working 195 | except Exception as e: 196 | logException(e, logger, verbose=verbose) 197 | logError("You probaly have to log to your gmail account and allow recent access. Or connect to https://accounts.google.com/b/0/DisplayUnlockCaptcha and click continue (WARNING: choose the right user after the link redirection by setting the user number of your browser (\"/b/0\" correspond to the first logged user in your browser))") 198 | 199 | def test1(): 200 | logger = Logger(tmpDir("teeeest") + "/aaa.txt", doPrint=False) 201 | for i in range(1000000): 202 | log(getRandomStr(), logger) 203 | 204 | if __name__ == '__main__': 205 | test1() 206 | 207 | 208 | 209 | 210 | 211 | 212 | -------------------------------------------------------------------------------- /systemtools/data/fr-lastnames.txt: -------------------------------------------------------------------------------- 1 | Martin 2 | Bernard 3 | Roux 4 | Thomas 5 | Petit 6 | Durand 7 | Michel 8 | Robert 9 | Richard 10 | Simon 11 | Moreau 12 | Dubois 13 | Blanc 14 | Laurent 15 | Girard 16 | Bertrand 17 | Garnier 18 | David 19 | Morel 20 | Guerin 21 | Fournier 22 | Roy 23 | Rousseau 24 | Andre 25 | Gautier 26 | Bonnet 27 | Lambert 28 | Henry 29 | Faure 30 | Mercier 31 | Vincent 32 | Chevalier 33 | Leroy 34 | Marchand 35 | Perrin 36 | Morin 37 | Masson 38 | Giraud 39 | Dupont 40 | Robin 41 | Nicolas 42 | Brun 43 | Mathieu 44 | Clement 45 | Lefebvre 46 | Fabre 47 | Barbier 48 | Francois 49 | Roussel 50 | Arnaud 51 | Gerard 52 | Aubert 53 | Duval 54 | Legrand 55 | Blanchard 56 | Brunet 57 | Lefevre 58 | Denis 59 | Breton 60 | Pierre 61 | Roche 62 | Paris 63 | Boyer 64 | Colin 65 | Fontaine 66 | Jean 67 | Bourgeois 68 | Gaillard 69 | Noel 70 | Dumas 71 | Picard 72 | Briand 73 | Lucas 74 | Rolland 75 | Joly 76 | Leclerc 77 | Renard 78 | Roger 79 | Dufour 80 | Olivier 81 | Gauthier 82 | Marie 83 | Guillou 84 | Guillaume 85 | Herve 86 | Leroux 87 | Bailly 88 | Dumont 89 | Philippe 90 | Boucher 91 | Renaud 92 | Deschamps 93 | Hamon 94 | Rey 95 | Royer 96 | Meunier 97 | Charpentier 98 | Baron 99 | Huet 100 | Riviere 101 | Carre 102 | Berger 103 | Prevost 104 | Aubry 105 | Vidal 106 | Lemoine 107 | Lemaire 108 | Lacroix 109 | Guillot 110 | Grand 111 | Adam 112 | Gall 113 | Guyot 114 | Perrot 115 | Goff 116 | Fleury 117 | Dupuy 118 | Daniel 119 | Julien 120 | Ollivier 121 | Hubert 122 | Millet 123 | Germain 124 | Louis 125 | Clerc 126 | Langlois 127 | Pelletier 128 | Benoit 129 | Besson 130 | Collet 131 | Moulin 132 | Renault 133 | Collin 134 | Jacquet 135 | Poirier 136 | Cousin 137 | Bouvier 138 | Caron 139 | Gillet 140 | Humbert 141 | Charles 142 | Reynaud 143 | Dupuis 144 | Gros 145 | Barthelemy 146 | Guichard 147 | Gay 148 | Marchal 149 | Lecomte 150 | Monnier 151 | Maillard 152 | France 153 | Gaultier 154 | Jacob 155 | Corre 156 | Leger 157 | Muller 158 | Morvan 159 | Pichon 160 | Chauvin 161 | Lebrun 162 | Mallet 163 | Remy 164 | Paul 165 | Barre 166 | Bertin 167 | Perret 168 | Buisson 169 | Chevallier 170 | Bigot 171 | Menard 172 | Meyer 173 | Marechal 174 | Jourdan 175 | Gilbert 176 | Antoine 177 | Regnier 178 | Coste 179 | Bourbon 180 | George 181 | Besnard 182 | Riou 183 | Labbe 184 | Thiers 185 | Floch 186 | Imbert 187 | Vallee 188 | Blanchet 189 | Lamy 190 | Camus 191 | Boulanger 192 | Pasquier 193 | Benoist 194 | Martel 195 | Delaunay 196 | Hugo 197 | Mahe 198 | Ferry 199 | Leblanc 200 | Lombard 201 | Texier 202 | Cordier 203 | Etienne 204 | Pons 205 | Favre 206 | Gaudin 207 | Carpentier 208 | Guillon 209 | Joubert 210 | Pascal 211 | Ricard 212 | Seguin 213 | Bouchet 214 | Salmon 215 | Vaillant 216 | Hebert 217 | Tanguy 218 | Poulain 219 | Hardy 220 | Didier 221 | Masse 222 | Michaud 223 | Ferrand 224 | Normand 225 | Combes 226 | Colas 227 | Courtois 228 | Albert 229 | Rousset 230 | Thibault 231 | Legendre 232 | Dupre 233 | Mace 234 | Perier 235 | Godard 236 | Gregoire 237 | Perrier 238 | Berthelot 239 | Delorme 240 | Guibert 241 | Poincare 242 | Sauvage 243 | Marion 244 | Girault 245 | Chartier 246 | Maurel 247 | Barbe 248 | Lesage 249 | Vial 250 | Allard 251 | Laval 252 | Alexandre 253 | President 254 | Launay 255 | Marin 256 | Bourdon 257 | Couturier 258 | Gras 259 | Berard 260 | Comte 261 | Verdier 262 | Maire 263 | Voisin 264 | Guillet 265 | Allain 266 | Marty 267 | Devaux 268 | Bazin 269 | Bon 270 | Bodin 271 | Jacques 272 | Merle 273 | Prigent 274 | Laurens 275 | Bruneau 276 | Bihan 277 | Leduc 278 | Foucher 279 | Bousquet 280 | Millot 281 | Lemaitre 282 | Potier 283 | Thierry 284 | Fouquet 285 | Coulon 286 | Chauvet 287 | Leveque 288 | Moal 289 | Regnault 290 | Bernier 291 | Ch 292 | Guyon 293 | Raymond 294 | Maurice 295 | Peron 296 | Chauveau 297 | Gilles 298 | Vasseur 299 | Leconte 300 | Page 301 | Clemenceau 302 | Jacquot 303 | Cornu 304 | Quere 305 | Jourdain 306 | Leclercq 307 | Tessier 308 | Rouxel 309 | Serre 310 | Georges 311 | Rossignol 312 | Guillemin 313 | Davy 314 | Lelievre 315 | Moine 316 | Champion 317 | Lenoir 318 | Rocher 319 | Benard 320 | Toussaint 321 | Valette 322 | Gueguen 323 | Beaumont 324 | Forest 325 | Geoffroy 326 | Forestier 327 | Lebreton 328 | Ingenieur 329 | Guen 330 | Loiseau 331 | Baudry 332 | Dupin 333 | Bonneau 334 | Leon 335 | Maurin 336 | Jamet 337 | Borgne 338 | Baudouin 339 | Legros 340 | Brunel 341 | Orleans 342 | Bouchard 343 | Laporte 344 | Auger 345 | Poisson 346 | Valentin 347 | Jullien 348 | Sabatier 349 | Blondeau 350 | Maillet 351 | Husson 352 | Lejeune 353 | Provost 354 | Maury 355 | Raynaud 356 | Valois 357 | Salaun 358 | Blondel 359 | Laroche 360 | Delmas 361 | Basset 362 | Gal 363 | Schneider 364 | Laugier 365 | Petitjean 366 | Schmitt 367 | Blin 368 | Mouton 369 | Prefet 370 | Brisson 371 | Constant 372 | Pineau 373 | Neveu 374 | Morand 375 | Turpin 376 | Castel 377 | Billard 378 | Jouan 379 | Berthier 380 | Claude 381 | Chretien 382 | Grange 383 | Parent 384 | Delahaye 385 | Jolly 386 | Aubin 387 | Martineau 388 | Bourgogne 389 | Duclos 390 | Belin 391 | Bras 392 | Poncet 393 | Rigaud 394 | Gervais 395 | Lemonnier 396 | Favier 397 | Drouet 398 | Charrier 399 | Lacombe 400 | Isnard 401 | Vallet 402 | Bouvet 403 | Bonnard 404 | Chapuis 405 | Cartier 406 | Martinet 407 | Tissot 408 | Carlier 409 | Pages 410 | Prost 411 | Chabert 412 | Meur 413 | Tournier 414 | Boutin 415 | Bourdin 416 | Cros 417 | Levy 418 | Bonhomme 419 | Duhamel 420 | Andrieu 421 | Guillemot 422 | Jolivet 423 | Lefort 424 | Klein 425 | Audibert 426 | Duchesne 427 | Jacquemin 428 | Grandjean 429 | Hamel 430 | Bois 431 | Baud 432 | Guy 433 | Parisot 434 | Laine 435 | Peltier 436 | Maitre 437 | Baudet 438 | Picot 439 | Arnoux 440 | Brossard 441 | Porte 442 | Bataille 443 | Roland 444 | Doucet 445 | Vernet 446 | Fortin 447 | Mangin 448 | Hugues 449 | Guillard 450 | Salle 451 | Granier 452 | Goujon 453 | Lagarde 454 | Bris 455 | Bureau 456 | Estienne 457 | Baudin 458 | Grenier 459 | Pottier 460 | Calvez 461 | Sicard 462 | Cheron 463 | Tellier 464 | Lepage 465 | Saulnier 466 | Herriot 467 | Charbonnier 468 | Prat 469 | Duret 470 | Merlin 471 | Huguet 472 | Marc 473 | Parmentier 474 | Arnould 475 | Mignot 476 | Rault 477 | Garcin 478 | Loubet 479 | Lorraine 480 | Berthet 481 | Lecoq 482 | Gicquel 483 | Chabot 484 | Cam 485 | Delattre 486 | Chambon 487 | Prieur 488 | Berre 489 | Bonnin 490 | Simonin 491 | Girardin 492 | Josse 493 | Dauphin 494 | Blot 495 | Armand 496 | Thiery 497 | Munier 498 | Jan 499 | Tardieu 500 | Gallois 501 | Weber 502 | Levesque 503 | Godefroy 504 | Lafont 505 | Rose 506 | Grosjean 507 | Evrard 508 | Goupil 509 | Duc 510 | Guiot 511 | Bonaparte 512 | Faivre 513 | Derrien 514 | Guilbert 515 | Hue 516 | Lafon 517 | Ribot 518 | Delacroix 519 | Saunier 520 | Cochet 521 | Chevrier 522 | Combe 523 | Boulay 524 | Lefeuvre 525 | Binet 526 | Borel 527 | Dubreuil 528 | Morice 529 | Lallemand 530 | Gibert 531 | Godin 532 | Carriere 533 | Coz 534 | Bonnefoy 535 | Geffroy 536 | Proust 537 | Caillaux 538 | Baudoin 539 | Dore 540 | Bastien 541 | Larcher 542 | Marquet 543 | Thebault 544 | Boudet 545 | Stephan 546 | Magnin 547 | Lacoste 548 | Janin 549 | Jegou 550 | Bocquet 551 | Delaporte 552 | Bremond 553 | Auffret 554 | Moret 555 | Achard 556 | Mounier 557 | Simonet 558 | Augier 559 | Villeneuve 560 | Janvier 561 | Berton 562 | Gentil 563 | Ferre 564 | Beraud 565 | Cosson 566 | Vacher 567 | Bourgoin 568 | Claudel 569 | Motte 570 | Thevenin 571 | Thevenet 572 | Bouquet 573 | Rouault 574 | Prudhomme 575 | Lefranc 576 | Leray 577 | Roques 578 | Lemarchand 579 | Lagrange 580 | Tardy 581 | Blaise 582 | Guizot 583 | Ferrier 584 | Blum 585 | Mary 586 | Foucault 587 | Savoie 588 | Conan 589 | Fort 590 | Gallet 591 | Long 592 | Lacour 593 | Boisson 594 | Cornet 595 | Jacquin 596 | Letellier 597 | Samson 598 | Felix 599 | Bourguignon 600 | Conte 601 | Bourhis 602 | Gillot 603 | Pelissier 604 | Chollet 605 | Bouillon 606 | Jacquier 607 | Navarre 608 | Guignard 609 | Laborde 610 | Viard 611 | Bellanger 612 | Granger 613 | Croix 614 | Anjou 615 | Vivier 616 | Robinet 617 | Beauvais 618 | Rivet 619 | Levasseur 620 | Jaures 621 | Gaubert 622 | Blandin 623 | Rouyer 624 | Colomb 625 | Ray 626 | Wagner 627 | Court 628 | Buffet 629 | Barthe 630 | Savary 631 | Delage 632 | Fevre 633 | Rambaud 634 | Jaouen 635 | Genin 636 | Pain 637 | Chardon 638 | Grimaud 639 | Metayer 640 | Tissier 641 | Puech 642 | Feraud 643 | Thiebaut 644 | Bresson 645 | Beau 646 | Esnault 647 | Gaucher 648 | Gosselin 649 | Bardin 650 | Guilloux 651 | Avril 652 | Cloarec 653 | Pepin 654 | Chapelle 655 | Brochard 656 | Prevot 657 | Boutet 658 | Maistre 659 | Bretagne 660 | Michon 661 | Bouche 662 | Rouge 663 | Berry 664 | Soulier 665 | Billon 666 | Mas 667 | Vigneron 668 | Tixier 669 | Monier 670 | Tavernier 671 | Dubost 672 | Poulet 673 | Jeanne 674 | Chatelain 675 | Drouin 676 | Gouin 677 | Piot 678 | Raoul 679 | Perron 680 | Astier 681 | Pommier 682 | Guitton 683 | Paillard 684 | Bail 685 | Godet 686 | Malet 687 | Gac 688 | Bontemps 689 | Ledoux 690 | Rougier 691 | Febvre 692 | Joseph 693 | Vannier 694 | Leblond 695 | Quentin 696 | Pierron 697 | Pinson 698 | Caillet 699 | Olive 700 | Bel 701 | Besse 702 | Jouve 703 | Bastard 704 | Nedelec 705 | Terrier 706 | Chaix 707 | Lhuillier 708 | Villain 709 | Deville 710 | Boisseau 711 | Michelet 712 | Payen 713 | Guegan 714 | Fouque 715 | Maintenon 716 | Jeune 717 | Salomon 718 | Pinard 719 | Cottin 720 | Froment 721 | Bonin 722 | Oger 723 | Rollet 724 | Naudin 725 | Bayle 726 | Bert 727 | Boulet 728 | Roman 729 | Girod 730 | Pujol 731 | Briere 732 | Leonard 733 | Marcel 734 | Brousse 735 | Dumoulin 736 | Lebeau 737 | Duchene 738 | Silvestre 739 | Verrier 740 | Pollet 741 | Sergent 742 | Brault 743 | Lemercier 744 | Lepine 745 | Champagne 746 | Lamoureux 747 | Bret 748 | Guiraud 749 | Constantin 750 | Allemand 751 | 752 | Mauger 753 | Sellier 754 | Pernot 755 | Pichard 756 | Collignon 757 | Lelong 758 | Amiot 759 | Bailleul 760 | Berenger 761 | Dreux 762 | Bidault 763 | Veron 764 | Viguier 765 | Even 766 | Remond 767 | Reymond 768 | Rouvier 769 | Lecocq 770 | Lefebure 771 | Verger 772 | Chauvel 773 | Charlot 774 | Pichot 775 | Montfort 776 | Delarue 777 | Perin 778 | Courtin 779 | Desbois 780 | Hamelin 781 | Reboul 782 | Delannoy 783 | Poirot 784 | Jardin 785 | Dugue 786 | Rochefort 787 | Marot 788 | Constans 789 | Millerand 790 | Sarrazin 791 | Lebon 792 | Girardot 793 | Brisset 794 | Villard 795 | Boivin 796 | Daumas 797 | Lay 798 | Marais 799 | Galland 800 | Richaud 801 | Calvet 802 | Richer 803 | Chabaud 804 | Foulon 805 | Montagne 806 | Charlet 807 | Pape 808 | Lamotte 809 | Nicolle 810 | Caille 811 | Vernier 812 | Ragot 813 | Descamps 814 | Huard 815 | Sage 816 | Mignon 817 | Rio 818 | Auvray 819 | Peyre 820 | Murat 821 | Portier 822 | Raynal 823 | Bisson 824 | Marteau 825 | Mille 826 | Cauvin 827 | Bellec 828 | Clavel 829 | Duchemin 830 | Thirion 831 | Tisserand 832 | Chenu 833 | Monin 834 | Monnet 835 | Barreau 836 | Chemin 837 | James 838 | Babin 839 | Fayolle 840 | Papin 841 | Jezequel 842 | Doumergue 843 | Viel 844 | Baillet 845 | Barthou 846 | Landry 847 | Lavigne 848 | Bergeron 849 | Louvet 850 | Caillaud 851 | Bastide 852 | Gontier 853 | Collot 854 | Salles 855 | Clavier 856 | Mugnier 857 | Roch 858 | Barret 859 | Leleu 860 | Rondeau 861 | Jarry 862 | Varin 863 | Bosc 864 | Coudray 865 | Oudin 866 | Revel 867 | Gabriel 868 | Mesnard 869 | Cadiou 870 | Bordier 871 | Pin 872 | Pillet 873 | Lavergne 874 | Porcher 875 | Patin 876 | Bec 877 | Nicol 878 | Berthon 879 | Chatillon 880 | Roze 881 | Sorel 882 | Guyard 883 | Gobert 884 | Haye 885 | Maillot 886 | Capet 887 | Hennequin 888 | Delisle 889 | Gravier 890 | Latour 891 | Blois 892 | Bourget 893 | Becker 894 | Rollin 895 | Faye 896 | Pierson 897 | Musset 898 | Charton 899 | Laisne 900 | Caillot 901 | Despres -------------------------------------------------------------------------------- /systemtools/printer.py: -------------------------------------------------------------------------------- 1 | 2 | # pew in st-venv python ~/Workspace/Python/Utils/SystemTools/systemtools/printer.py 3 | 4 | import sys 5 | from enum import Enum 6 | from systemtools.logger import * 7 | from systemtools.number import truncateFloat 8 | from collections import OrderedDict 9 | import collections 10 | import re 11 | import numpy as np 12 | 13 | 14 | class COLOR: 15 | purple = '\033[95m' 16 | cyan = '\033[96m' 17 | darkcyan = '\033[36m' 18 | blue = '\033[94m' 19 | green = '\033[92m' 20 | yellow = '\033[93m' 21 | red = '\033[91m' 22 | bold = '\033[1m' 23 | underline = '\033[4m' 24 | end = '\033[0m' 25 | 26 | 27 | BTYPE = Enum("BTYPE", "dict list set string type none maxdepth number object tuple ndarray") 28 | 29 | 30 | def __bold(s): 31 | """ 32 | This function return a string wrapped with bold tokens if and only if the script is run under tty or notebook: 33 | """ 34 | # isNotebook = '__file__' not in locals() 35 | isTTY = sys.stdin.isatty() 36 | if isTTY: 37 | return COLOR.bold + s + COLOR.end 38 | else: 39 | return s 40 | 41 | def reduceBlank(text, keepNewLines=False): 42 | """ 43 | Strip a string and reduce all blank space to a unique space. If you set keepNewLines as True, it will keep a unique '\n' at each blank space which contains a '\n' or a '\r' 44 | """ 45 | if text is None: 46 | return None 47 | text = text.strip() 48 | if not keepNewLines: 49 | return re.sub(r'\s+', ' ', text) 50 | else: 51 | text = re.sub(r'\r', '\n', text) 52 | text = re.sub(r'\s*\n+\s*', '\n', text) 53 | text = re.sub(r'[ \t\f\v]+', ' ', text) 54 | return text 55 | 56 | 57 | def bp(obj, level='auto', logger=None, verbose=True, **kwargs): 58 | try: 59 | if not (isinstance(level, str) or isinstance(level, int)): 60 | if logger is None: 61 | level, logger = "auto", level 62 | elif isinstance(logger, str) or isinstance(logger, int): 63 | level, logger = logger, level 64 | else: 65 | raise Exception("You must give level as an int or str and logger as a logger or as an object which has logger in its attributes") 66 | text = b(obj, level=level, logger=logger, verbose=verbose, **kwargs) 67 | log(text, logger, verbose=verbose) 68 | except Exception as e1: 69 | logException(e1, logger, location="bp", verbose=verbose) 70 | 71 | def b(obj, level='auto', logger=None, verbose=True, **kwargs): 72 | """ 73 | levels: 74 | * 0 --> only schema, very minimized... 75 | * 1 --> only schema 76 | * 2 --> very minimized 77 | * 3 --> minimized 78 | * 4 --> few minimized 79 | * 5 --> full 80 | * auto --> choose automatically params on-the-fly (~ level 2) 81 | """ 82 | try: 83 | if isinstance(level, int) and level < 0: 84 | level = 0 85 | if isinstance(level, int) and level > 5: 86 | level = 5 87 | if level == 'auto': 88 | level = 2 89 | if level <= 1: 90 | if "schemaOnly" not in kwargs: 91 | kwargs["schemaOnly"] = True 92 | if level <= 2: 93 | if "maxDictLen" not in kwargs: 94 | kwargs["maxDictLen"] = 10 95 | if "maxStringLen" not in kwargs: 96 | kwargs["maxStringLen"] = 100 97 | if "maxIterableLen" not in kwargs: 98 | kwargs["maxIterableLen"] = 4 99 | if "maxDecimals" not in kwargs: 100 | kwargs["maxDecimals"] = 2 101 | if level <= 3: 102 | if "maxDictLen" not in kwargs: 103 | kwargs["maxDictLen"] = 15 104 | if "addQuotes" not in kwargs: 105 | kwargs["addQuotes"] = False 106 | if "doReduceBlank" not in kwargs: 107 | kwargs["doReduceBlank"] = True 108 | if "maxStringLen" not in kwargs: 109 | kwargs["maxStringLen"] = 150 110 | if "maxIterableLen" not in kwargs: 111 | kwargs["maxIterableLen"] = 10 112 | if "maxDecimals" not in kwargs: 113 | kwargs["maxDecimals"] = 4 114 | if level <= 4: 115 | if "maxStringLen" not in kwargs: 116 | kwargs["maxStringLen"] = 400 117 | if "maxIterableLen" not in kwargs: 118 | kwargs["maxIterableLen"] = 20 119 | if "maxDictLen" not in kwargs: 120 | kwargs["maxDictLen"] = 20 121 | text = beautif(obj, **kwargs) 122 | if text.startswith("\n"): 123 | text = text[1:] 124 | return text 125 | except Exception as e: 126 | logException(e, location="b", logger=logger, verbose=verbose) 127 | 128 | 129 | def beautif(*args, tab=" ", **kwargs): 130 | (values, btype) = __beautif(*args, tab=tab, **kwargs) 131 | return innerMultilines(values, btype, tab, "") 132 | 133 | def __beautif\ 134 | ( 135 | obj, 136 | tab=" ", 137 | depth=0, 138 | width=None, 139 | maxStringLen=None, 140 | maxIterableLen=None, 141 | maxDictLen=None, 142 | schemaOnly=False, 143 | addQuotes=True, 144 | doReduceBlank=False, 145 | maxDepth=None, 146 | doSort=True, # on sets and dicts 147 | maxDecimals=None, 148 | # TODO unknownObjectsToType=False, # type or __repr__ or __str__ 149 | # TODO autoDetokenize=False, 150 | ): 151 | # try: 152 | # Not yet implemented 153 | try: 154 | assert width is None 155 | assert not schemaOnly 156 | except: 157 | raise Exception("Not yet implemented") 158 | # We construct recursive kwargs: 159 | recKwargs = locals() 160 | del recKwargs["obj"] 161 | recKwargs["depth"] += 1 162 | # We construct the tabulation: 163 | shift = tab * depth # if depth > 0 else "" 164 | # We calculate the reamining space: 165 | if width is None: 166 | remainingSpace = None 167 | else: 168 | remainingSpace = width - len(shift) 169 | # Now if test the obj type: 170 | if depth == maxDepth: 171 | return ([""], BTYPE.maxdepth) 172 | elif obj is None: 173 | return ([str(obj)], BTYPE.none) 174 | elif isinstance(obj, str): 175 | if schemaOnly: 176 | return ([type(obj)], BTYPE.none) 177 | if doReduceBlank: 178 | obj = reduceBlank(obj, keepNewLines=True) 179 | if maxStringLen is not None: 180 | obj = obj[:maxStringLen] 181 | # if remainingSpace is not None and len(obj) >= remainingSpace: 182 | # objList = [""] 183 | # objListIndex = 0 184 | # for i in range(len(obj)): 185 | # if len(objList[objListIndex]) >= remainingSpace: 186 | # objList.append("") 187 | # objListIndex += 1 188 | # objList[objListIndex] += obj[i] 189 | # newObj = "" 190 | # for current in objList: 191 | # newObj += shift + current + "\n" 192 | # obj = newObj[:-1] 193 | return ([obj], BTYPE.string) 194 | elif isinstance(obj, dict) or isinstance(obj, OrderedDict): 195 | if isinstance(obj, OrderedDict): 196 | obj = dict(obj) 197 | result = [] 198 | result.append("{") 199 | quote = "'" if addQuotes else "" 200 | if doSort: 201 | keys = sorted(list(obj.keys()), key=str) 202 | else: 203 | keys = obj.keys() 204 | if maxDictLen is not None and len(keys) > maxDictLen: 205 | amount = int(maxDictLen / 2) 206 | if amount == 0: 207 | amount = 1 208 | keysParts = [keys[:amount], keys[-amount:]] 209 | else: 210 | keysParts = [keys] 211 | for keysPartsIndex in range(len(keysParts)): 212 | keysPart = keysParts[keysPartsIndex] 213 | for key in keysPart: 214 | value = obj[key] 215 | (values, btype) = __beautif(value, **recKwargs) 216 | values = innerMultilines(values, btype, tab, shift + tab) 217 | result.append(quote + __bold(str(key)) + quote + ": " + values + ",") 218 | if len(keysParts) > 1 and keysPartsIndex != len(keysParts) - 1: 219 | result.append("...,") 220 | # We remove the comma: 221 | try: 222 | if result[-1][-1] == ",": 223 | result[-1] = result[-1][:-1] 224 | except: pass 225 | # We close the dict: 226 | result.append('}') 227 | return (result, BTYPE.dict) 228 | elif isinstance(obj, list) or isinstance(obj, set) or isinstance(obj, tuple) or isinstance(obj, collections.abc.KeysView): 229 | if isinstance(obj, collections.abc.KeysView): 230 | obj = set(obj) 231 | isSet = isinstance(obj, set) 232 | isTuple = isinstance(obj, tuple) 233 | isList = isinstance(obj, list) 234 | if isSet and doSort: 235 | obj = sorted(obj, key=str) 236 | parts = [obj] 237 | if maxIterableLen is not None and len(obj) > maxIterableLen: 238 | amount = int(maxIterableLen / 2) 239 | if amount == 0: 240 | amount = 1 241 | parts = [obj[:amount], obj[-amount:]] 242 | result = [] 243 | if isSet: 244 | result.append("{") 245 | elif isTuple: 246 | result.append("(") 247 | else: 248 | result.append("[") 249 | partIndex = 0 250 | for partIndex in range(len(parts)): 251 | part = parts[partIndex] 252 | for value in part: 253 | (values, btype) = __beautif(value, **recKwargs) 254 | values = innerMultilines(values, btype, tab, shift + tab) 255 | result.append(values + ",") 256 | if len(parts) > 1 and partIndex != len(parts) - 1: 257 | result.append("...,") 258 | # We remove the comma: 259 | try: 260 | if result[-1][-1] == ",": 261 | result[-1] = result[-1][:-1] 262 | except: pass 263 | # We close the iterable: 264 | if isSet: 265 | result.append("}") 266 | return (result, BTYPE.set) 267 | elif isTuple: 268 | result.append(")") 269 | return (result, BTYPE.tuple) 270 | else: 271 | result.append("]") 272 | return (result, BTYPE.list) 273 | elif isinstance(obj, int) or isinstance(obj, float): 274 | if isinstance(obj, float) and maxDecimals is not None: 275 | obj = truncateFloat(obj, maxDecimals) 276 | return ([str(obj)], BTYPE.number) 277 | elif isinstance(obj, bool): 278 | return ([str(obj)], BTYPE.boolean) 279 | elif isinstance(obj, np.ndarray): 280 | return (str(obj).split("\n"), BTYPE.ndarray) 281 | else: 282 | text = str(obj) 283 | if doReduceBlank: 284 | text = reduceBlank(text, keepNewLines=True) 285 | if maxStringLen is not None and len(text) > maxStringLen: 286 | return ([text[:maxStringLen] + "..."], BTYPE.object) 287 | else: 288 | return ([text], BTYPE.object) 289 | 290 | 291 | # except Exception as e: 292 | #  print(e) 293 | 294 | def innerMultilines(values, btype, tab, shift): 295 | if values is None or len(values) == 0: 296 | return "" 297 | elif len(values) == 1: 298 | return values[0] 299 | else: 300 | hasOnlyLittleElements = True 301 | for current in values: 302 | if len(current) > 50: 303 | hasOnlyLittleElements = False 304 | break 305 | if hasOnlyLittleElements and not btype == BTYPE.ndarray: 306 | result = " ".join(values) 307 | return result 308 | else: 309 | # if values[0][0] == "\n": 310 | # values[0] = values[1:] 311 | result = "" 312 | if len(values[0]) == 1: 313 | result += "\n" + shift + values[0] + "\n" 314 | else: 315 | result += values[0] + "\n" 316 | doMiddleTab = btype in [BTYPE.list, BTYPE.dict, BTYPE.set, BTYPE.tuple] 317 | if doMiddleTab: 318 | for i in range(1, len(values) - 1): 319 | values[i] = tab + values[i] 320 | for i in range(1, len(values)): 321 | current = values[i] 322 | result += shift + current + "\n" 323 | result = result[:-1] 324 | result = result.replace("\n" + shift + tab + "\n", "\n") # TODO find a clean way to handle this 325 | return result 326 | 327 | 328 | def test1(): 329 | obj = \ 330 | { 331 | 'uuu': [1, 2, 3, 4, 5, 6, {"y", "r", "e", "o", "p", "q"}, [5] * 100], 332 | 'eee': "blah " * 200, 333 | 'aaa': "bleh " * 400, 334 | 'ccc': "bluh " * 4, 335 | 'ddd': {"u": "e", "i": {"u": "e", "i": "e", "i": {"u": "e " * 4, "i": "e " * 4}}}, 336 | } 337 | print(beautif(obj, maxIterableLen=5, maxDictLen=5, maxStringLen=100)) 338 | 339 | def test2(): 340 | from datatools.jsonutils import NDJson 341 | row = None 342 | for row in NDJson("/home/hayj/tmp/mbti-datasets/mbti-dataset-2019.06.21-20.54/0.ndjson.bz2"): 343 | break 344 | bp(row, maxDepth=3) 345 | 346 | 347 | 348 | 349 | if __name__ == '__main__': 350 | test3() -------------------------------------------------------------------------------- /systemtools/location.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import subprocess 4 | import time 5 | import psutil 6 | import inspect 7 | import os 8 | import glob 9 | import sys 10 | import re 11 | import random 12 | import getpass 13 | from pathlib import Path 14 | import socket 15 | from systemtools.number import digitalizeIntegers 16 | from operator import itemgetter 17 | try: 18 | import pwd 19 | except: pass 20 | from systemtools import config as systConf 21 | 22 | def decomposePath(path): 23 | """ 24 | :example: 25 | >>> decomposePath(None) 26 | >>> decomposePath("") 27 | >>> decomposePath(1) 28 | >>> decomposePath("truc") 29 | ('', 'truc', '', 'truc') 30 | >>> decomposePath("truc.txt") 31 | ('', 'truc', 'txt', 'truc.txt') 32 | >>> decomposePath("/home/truc.txt") 33 | ('/home/', 'truc', 'txt', 'truc.txt') 34 | >>> decomposePath("/home/truc.txt.bz2") 35 | ('/home/', 'truc.txt', 'bz2', 'truc.txt.bz2') 36 | >>> decomposePath("/truc.txt.bz2") 37 | ('/', 'truc.txt', 'bz2', 'truc.txt.bz2') 38 | >>> decomposePath("./truc.txt.bz2") 39 | ('./', 'truc.txt', 'bz2', 'truc.txt.bz2') 40 | >>> decomposePath(".truc.txt.bz2") 41 | ('', '.truc.txt', 'bz2', '.truc.txt.bz2') 42 | """ 43 | if path is None or type(path) is not str or len(path) == 0: 44 | return None 45 | filenameExt = path.split("/")[-1] 46 | dir = path[0:-len(filenameExt)] 47 | filename = ".".join(filenameExt.split(".")[0:-1]) 48 | ext = filenameExt.split(".")[-1] 49 | if len(filename) == 0 and len(ext) > 0: 50 | filename, ext = ext, filename 51 | return (dir, filename, ext, filenameExt) 52 | 53 | def decomposePath2(path): 54 | """ 55 | :example: 56 | >>> decomposePath(None) 57 | >>> decomposePath("") 58 | >>> decomposePath(1) 59 | >>> decomposePath("truc") 60 | ('', 'truc', '', 'truc') 61 | >>> decomposePath("truc.txt") 62 | ('', 'truc', 'txt', 'truc.txt') 63 | >>> decomposePath("/home/truc.txt") 64 | ('/home/', 'truc', 'txt', 'truc.txt') 65 | >>> decomposePath("/home/truc.txt.bz2") 66 | ('/home/', 'truc.txt', 'bz2', 'truc.txt.bz2') 67 | >>> decomposePath("/truc.txt.bz2") 68 | ('/', 'truc.txt', 'bz2', 'truc.txt.bz2') 69 | >>> decomposePath("./truc.txt.bz2") 70 | ('./', 'truc.txt', 'bz2', 'truc.txt.bz2') 71 | >>> decomposePath(".truc.txt.bz2") 72 | ('', '.truc.txt', 'bz2', '.truc.txt.bz2') 73 | """ 74 | if path is None or type(path) is not str or len(path) == 0: 75 | return None 76 | filenameExt = path.split("/")[-1] 77 | dir = path[0:-len(filenameExt)] 78 | if len(dir) > 1 and dir[-1] == "/": 79 | dir = dir[:-1] 80 | filename = ".".join(filenameExt.split(".")[0:-1]) 81 | ext = filenameExt.split(".")[-1] 82 | if len(filename) == 0 and len(ext) > 0: 83 | filename, ext = ext, filename 84 | return (dir, filename, ext, filenameExt) 85 | 86 | def enhanceDir(path): 87 | if path[-1] != '/': 88 | path += '/' 89 | return path 90 | 91 | def removeLastSlash(text): 92 | if isinstance(text, str): 93 | if text[-1] == '/': 94 | return text[0:-1] 95 | return text 96 | 97 | def getCurrentDir(): 98 | return os.getcwd() 99 | 100 | def pathToAbsolute(path): 101 | if len(path) > 0 and path[0] != "/": 102 | path = getCurrentDir() + "/" + path 103 | return path 104 | 105 | def isFile(filePath): 106 | filePath = pathToAbsolute(filePath) 107 | # print filePath 108 | return os.path.isfile(filePath) 109 | 110 | def isDir(dirPath): 111 | dirPath = pathToAbsolute(dirPath) 112 | return os.path.isdir(dirPath) 113 | 114 | def getDir(filePath): 115 | return os.path.dirname(os.path.abspath(filePath)) 116 | 117 | def parentDir(*args, **kwargs): 118 | return getParentDir(*args, **kwargs) 119 | def getParentDir(path, depth=1): 120 | for i in range(depth): 121 | path = os.path.abspath(os.path.join(path, os.pardir)) 122 | return path 123 | 124 | def absPath(path): 125 | return os.path.abspath(path) 126 | 127 | # Deprecated: 128 | def getRootDirectory(*args, **kwargs): 129 | return getExecDirectory(*args, **kwargs) 130 | def getExecDir(*args, **kwargs): 131 | return getExecDirectory(*args, **kwargs) 132 | def execPath(*args, **kwargs): 133 | return getExecDirectory(*args, **kwargs) 134 | def execDir(*args, **kwargs): 135 | return getExecDirectory(*args, **kwargs) 136 | def getExecDirectory(_file_=None): 137 | """ 138 | Get the directory of the root execution file 139 | Can help: http://stackoverflow.com/questions/50499/how-do-i-get-the-path-and-name-of-the-file-that-is-currently-executing 140 | For eclipse user with unittest or debugger, the function search for the correct folder in the stack 141 | You can pass __file__ (with 4 underscores) if you want the caller directory 142 | """ 143 | # If we don't have the __file__ : 144 | if _file_ is None: 145 | # We get the last : 146 | rootFile = inspect.stack()[-1][1] 147 | folder = os.path.abspath(rootFile) 148 | # If we use unittest : 149 | if ("/pysrc" in folder) and ("org.python.pydev" in folder): 150 | previous = None 151 | # We search from left to right the case.py : 152 | for el in inspect.stack(): 153 | currentFile = os.path.abspath(el[1]) 154 | if ("unittest/case.py" in currentFile) or ("org.python.pydev" in currentFile): 155 | break 156 | previous = currentFile 157 | folder = previous 158 | # We return the folder : 159 | return os.path.dirname(folder) 160 | else: 161 | # We return the folder according to specified __file__ : 162 | return os.path.dirname(os.path.realpath(_file_)) 163 | 164 | def rtmpDir(subDir=None): 165 | # Then we modified subDir if needed: 166 | if subDir is None: 167 | subDir = "" 168 | elif not subDir.startswith("/"): 169 | subDir = "/" + subDir 170 | # We return the rtmp dir 171 | if isDir("/rtmp"): 172 | theDir = "/rtmp" + subDir 173 | return "/rtmp" 174 | elif isDir("/tmp"): 175 | theDir = "/tmp/hayj" + subDir 176 | else: 177 | raise Exception("No root tmp dir found") 178 | os.makedirs(theDir, exist_ok=True) 179 | return theDir 180 | 181 | def getWorkingDirectory(*args, **kwargs): 182 | return tmpDir(*args, **kwargs) 183 | def getTmpDir(*args, **kwargs): 184 | return tmpDir(*args, **kwargs) 185 | def tmpPath(*args, **kwargs): 186 | return tmpDir(*args, **kwargs) 187 | def getTmpPath(*args, **kwargs): 188 | return tmpDir(*args, **kwargs) 189 | def tmpDir(_file_=None, subDir=None): 190 | """ 191 | _file_ and subDir can be switched 192 | The rule is _file_ must finish by ".py" 193 | """ 194 | # First we switch both parameters if needed: 195 | if _file_ is not None and _file_[-3:] != ".py": 196 | subDir, _file_ = _file_, subDir 197 | # Then we modified subDir if needed: 198 | if subDir is None: 199 | subDir = "" 200 | elif not subDir.startswith("/"): 201 | subDir = "/" + subDir 202 | # Finally we get the root path: 203 | if _file_ is None: 204 | rootPath = homeDir() 205 | else: 206 | rootPath = execDir(_file_) 207 | # And we get the tmp directory: 208 | if systConf.defaultTmpDir is not None: 209 | workingPath = systConf.defaultTmpDir + subDir 210 | else: 211 | workingPath = rootPath + "/tmp" + subDir 212 | # Add the tmp directory if not exists : 213 | os.makedirs(workingPath, exist_ok=True) 214 | return workingPath 215 | 216 | # Deprecated: 217 | def getUtilDirectory(): 218 | return os.path.dirname(os.path.realpath(__file__)); 219 | 220 | class GlobSortEnum(): 221 | ( 222 | AUTO, 223 | MTIME, 224 | NAME, 225 | SIZE, 226 | NUMERICAL_NAME 227 | ) = list(range(5)) 228 | 229 | 230 | def sortedGlob(regex, caseSensitive=True, sortBy=GlobSortEnum.AUTO, reverse=False): 231 | """ 232 | See the README 233 | 234 | :params: 235 | regex : string 236 | the pattern used to find files or folders 237 | caseSensitive : boolean 238 | set it as False if you don't want to take care of the case 239 | sortBy : enum item 240 | can be GlobSortEnum., 241 | the last one is the same as name but take into account the last number in the given path numbers 242 | (e.g. test1.txt < test10.txt) 243 | reverse : boolean 244 | set it as True if you want to reverse the order 245 | """ 246 | def getFileNum(fileName): 247 | """ 248 | :example: 249 | >>> getFileNum(None) 250 | >>> getFileNum("truc") 251 | >>> getFileNum("truc.md") 252 | >>> getFileNum("10truc2.4md") 253 | 10 254 | >>> getFileNum("505.bz2") 255 | 505 256 | >>> getFileNum("505.bz2") 257 | 505 258 | """ 259 | if fileName is None or len(fileName) == 0: 260 | return None 261 | result = re.findall("([0-9]+)", fileName) 262 | if result is not None: 263 | try: 264 | theInt = result[-1] 265 | return int(theInt) 266 | except IndexError as e: 267 | return None 268 | return None 269 | 270 | # case insensitive glob function : 271 | def insensitiveGlob(pattern): 272 | def either(c): 273 | return '[%s%s]'%(c.lower(), c.upper()) if c.isalpha() else c 274 | return glob.glob(''.join(map(either, pattern))) 275 | 276 | # Handle case insentive param : 277 | if caseSensitive: 278 | paths = glob.glob(regex) 279 | else: 280 | paths = insensitiveGlob(regex) 281 | 282 | # Sort the result : 283 | if sortBy == GlobSortEnum.AUTO: 284 | # First we replace all integers by "000000012" for "12" for example: 285 | totalDigits = 100 286 | data = [] 287 | for text in paths: 288 | data.append((text, digitalizeIntegers(text))) 289 | data = sorted(data, key=itemgetter(1)) 290 | paths = [x for x, y in data] 291 | elif sortBy == GlobSortEnum.NAME: 292 | print("DEPRECATED GlobSortEnum.NAME, use GlobSortEnum.AUTO instead") 293 | paths.sort(reverse=reverse) 294 | elif sortBy == GlobSortEnum.MTIME: 295 | paths.sort(key=os.path.getmtime, reverse=reverse) 296 | elif sortBy == GlobSortEnum.SIZE: 297 | paths.sort(key=os.path.getsize, reverse=reverse) 298 | elif sortBy == GlobSortEnum.NUMERICAL_NAME: 299 | print("DEPRECATED GlobSortEnum.NUMERICAL_NAME, use GlobSortEnum.AUTO instead") 300 | paths.sort(key=getFileNum, reverse=reverse) 301 | 302 | return list(paths) 303 | 304 | 305 | def homePath(*args, **kwargs): 306 | return homeDir(*args, **kwargs) 307 | 308 | def nosaveDir(): 309 | if isDir("/NoSave"): 310 | return "/NoSave" 311 | elif isDir(homeDir() + "/NoSave"): 312 | return homeDir() + "/NoSave" 313 | else: 314 | print("No NoSave dir found.") 315 | return tmpDir() 316 | 317 | def homeDir(): 318 | if isDir("/hosthome"): 319 | return "/hosthome" 320 | else: 321 | return str(Path.home()) 322 | 323 | 324 | def getDataPath(*args, **kwargs): 325 | return dataPath(*args, **kwargs) 326 | def getDataDir(*args, **kwargs): 327 | return dataPath(*args, **kwargs) 328 | def dataDir(*args, **kwargs): 329 | return dataPath(*args, **kwargs) 330 | def dataPath(defaultDirName="Data"): 331 | def isHostname(hostname): 332 | return socket.gethostname().startswith(hostname) 333 | if isDir("/NoSave"): 334 | return "/NoSave/Data" 335 | elif isDir(homeDir() + "/NoSave"): 336 | return "/users/lahdak-nosave/hayj/" + defaultDirName 337 | else: 338 | return homeDir() + "/" + defaultDirName 339 | 340 | def sortedWalk(): 341 | pass # TODO 342 | 343 | 344 | 345 | 346 | def owner(filename): 347 | try: 348 | return pwd.getpwuid(os.stat(filename).st_uid).pw_name 349 | except Exception as e: 350 | return None 351 | 352 | if __name__ == '__main__': 353 | # print(tmpDir(subDir="test")) 354 | # print(sortedGlob(dataDir() + "/*")) 355 | 356 | 357 | for current in sortedGlob("/tmp/*"): 358 | print(current) 359 | print(owner(current)) 360 | print() 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | -------------------------------------------------------------------------------- /systemtools/test/basics.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # pew in systemtools-venv python ./test/basics.py 3 | 4 | import os 5 | import sys 6 | sys.path.append('../') 7 | 8 | import unittest 9 | import doctest 10 | from systemtools import basics 11 | from systemtools.basics import * 12 | 13 | # The level allow the unit test execution to choose only the top level test 14 | mini = 10 15 | maxi = 15 16 | assert mini <= maxi 17 | 18 | print("==============\nStarting unit tests...") 19 | 20 | if mini <= 0 <= maxi: 21 | class DocTest(unittest.TestCase): 22 | def testDoctests(self): 23 | """Run doctests""" 24 | doctest.testmod(basics) 25 | 26 | if mini <= 1 <= maxi: 27 | class Test1(unittest.TestCase): 28 | def test1(self): 29 | pass 30 | 31 | if mini <= 2 <= maxi: 32 | class Test2(unittest.TestCase): 33 | def test1(self): 34 | d1 = [1, 2, 3] 35 | d2 = [2, 3, 4] 36 | d3 = [10, 11, 12] 37 | d4 = [1, 10] 38 | 39 | self.assertTrue(intersection([d1, d2]) == [2, 3]) 40 | self.assertTrue(intersection([d1, d4]) == [1]) 41 | self.assertTrue(intersection([d1, d3]) == []) 42 | self.assertTrue(intersection([d1, None]) == []) 43 | self.assertTrue(intersection([None, None]) == []) 44 | self.assertTrue(intersection(None) == []) 45 | self.assertTrue(intersection([None]) == []) 46 | self.assertTrue(intersection([d1, d2, d3, d4]) == []) 47 | self.assertTrue(intersection([d1, d2, d4]) == []) 48 | self.assertTrue(intersection([d1 + d2, d4]) == [1]) 49 | 50 | def test2(self): 51 | d1 = ["aa", "bb"] 52 | d2 = "aa bb cc" 53 | d3 = ["cc", "ddd"] 54 | d4 = "aa dd cc" 55 | 56 | self.assertTrue(sorted(intersection([d1, d2])) == sorted(["bb", "aa"])) 57 | self.assertTrue(intersection([d3, d4]) == ["cc"]) 58 | self.assertTrue(intersection([d1, d3, d2]) == []) 59 | self.assertTrue(intersection([d1, d2, d4]) == ["aa"]) 60 | 61 | if mini <= 3 <= maxi: 62 | class Test3(unittest.TestCase): 63 | def test1(self): 64 | l = list(range(100)) 65 | result = chunks(l, 6) 66 | printLTS(result) 67 | self.assertTrue(len(result) == math.ceil(100/6)) 68 | result = split(l, 6) 69 | printLTS(result) 70 | self.assertTrue(len(result) == 6) 71 | 72 | 73 | l = list(range(7)) 74 | result = chunks(l, 6) 75 | printLTS(result) 76 | self.assertTrue(len(result) == 2) 77 | self.assertTrue(len(result[0]) == 6) 78 | self.assertTrue(len(result[1]) == 1) 79 | result = split(l, 6) 80 | printLTS(result) 81 | self.assertTrue(len(result) == 6) 82 | 83 | l = [] 84 | result = chunks(l, 1) 85 | self.assertTrue(len(result) == 0) 86 | result = chunks(l, 2) 87 | self.assertTrue(len(result) == 0) 88 | result = split(l, 1) 89 | self.assertTrue(len(result) == 1) 90 | result = split(l, 2) 91 | self.assertTrue(len(result) == 2) 92 | 93 | l = None 94 | self.assertTrue(split(l, 1) == []) 95 | self.assertTrue(chunks(l, 1) == []) 96 | 97 | l = [1] 98 | result = chunks(l, 1) 99 | self.assertTrue(len(result) == 1) 100 | self.assertTrue(len(result[0]) == 1) 101 | result = chunks(l, 2) 102 | self.assertTrue(len(result) == 1) 103 | self.assertTrue(result == [[1]]) 104 | result = split(l, 1) 105 | self.assertTrue(len(result) == 1) 106 | self.assertTrue(result == [[1]]) 107 | result = split(l, 2) 108 | self.assertTrue(len(result) == 2) 109 | self.assertTrue(result == [[1], []]) 110 | 111 | l = list(range(10)) 112 | result = chunks(l, 12) 113 | printLTS(result) 114 | self.assertTrue(len(result) == 1) 115 | result = split(l, 12) 116 | self.assertTrue(len(result) == 12) 117 | 118 | 119 | l = [1] 120 | result = split(l, 1) 121 | self.assertTrue(len(result) == 1) 122 | l = [1] 123 | result = split(l, 6) 124 | self.assertTrue(len(result) == 6) 125 | 126 | if mini <= 4 <= maxi: 127 | class Test4(unittest.TestCase): 128 | def test1(self): 129 | o1 = \ 130 | { 131 | "summary_detail": \ 132 | { 133 | "language": None, 134 | "test": 1, 135 | }, 136 | "link": "http://...", 137 | } 138 | 139 | self.assertTrue( 140 | getDictSubElement(o1, ["summary_detail", "language"]) == None) 141 | self.assertTrue( 142 | getDictSubElement(o1, ["summary_detail", "test"]) == 1) 143 | self.assertTrue( 144 | getDictSubElement(o1, ["summary_detail", "a"]) == None) 145 | self.assertTrue( 146 | getDictSubElement(o1, ["summary", "test"]) == None) 147 | self.assertTrue( 148 | getDictSubElement(o1, ["link", "test"]) == None) 149 | self.assertTrue( 150 | getDictSubElement(o1, ["link"]) == "http://...") 151 | 152 | o1 = \ 153 | { 154 | "summary_detail": \ 155 | { 156 | "language": {"test": 2}, 157 | "test": 1, 158 | }, 159 | "link": "http://...", 160 | } 161 | self.assertTrue( 162 | getDictSubElement(o1, ["summary_detail", "language"]) == {"test": 2}) 163 | self.assertTrue( 164 | getDictSubElement(o1, ["summary_detail", "language", "test"]) == 2) 165 | 166 | 167 | if mini <= 5 <= maxi: 168 | class Test5(unittest.TestCase): 169 | def test1(self): 170 | t = "aa \n uu.\roo\trr " 171 | reducedT = reduceBlank(t, keepNewLines=False) 172 | knlReducedT = reduceBlank(t, keepNewLines=True) 173 | self.assertTrue(reducedT == "aa uu. oo rr") 174 | self.assertTrue(knlReducedT == "aa\nuu.\noo rr") 175 | 176 | t = "\n\n\naa.bb oo\n" 177 | reducedT = reduceBlank(t, keepNewLines=False) 178 | knlReducedT = reduceBlank(t, keepNewLines=True) 179 | self.assertTrue(reducedT == "aa.bb oo") 180 | self.assertTrue(knlReducedT == "aa.bb oo") 181 | 182 | 183 | if mini <= 6 <= maxi: 184 | class Test6(unittest.TestCase): 185 | def test1(self): 186 | l = list(range(10)) 187 | l = splitMaxSized(l, 5) 188 | self.assertTrue(len(l) == 2) 189 | self.assertTrue(len(l[0]) == 5) 190 | self.assertTrue(len(l[1]) == 5) 191 | 192 | l = list(range(9)) 193 | l = splitMaxSized(l, 5) 194 | self.assertTrue(len(l) == 2) 195 | self.assertTrue(len(l[0]) == 5) 196 | self.assertTrue(len(l[1]) == 4) 197 | 198 | l = list(range(11)) 199 | l = splitMaxSized(l, 5) 200 | self.assertTrue(len(l) == 3) 201 | self.assertTrue(len(l[0]) == 4) 202 | self.assertTrue(len(l[1]) == 4) 203 | self.assertTrue(len(l[2]) == 3) 204 | 205 | l = list(range(1)) 206 | l = splitMaxSized(l, 5) 207 | self.assertTrue(len(l) == 1) 208 | self.assertTrue(len(l[0]) == 1) 209 | 210 | l = list(range(11)) 211 | l = splitMaxSized(l, 0) 212 | self.assertTrue(len(l) == 1) 213 | self.assertTrue(len(l[0]) == 11) 214 | 215 | l = list(range(11)) 216 | l = splitMaxSized(l, 1) 217 | self.assertTrue(len(l) == 11) 218 | self.assertTrue(len(l[0]) == 1) 219 | self.assertTrue(len(l[1]) == 1) 220 | self.assertTrue(len(l[2]) == 1) 221 | 222 | if mini <= 8 <= maxi: 223 | class Test8(unittest.TestCase): 224 | def test1(self): 225 | l = ["a", "a", "b", "c", "ddd", "c "] 226 | self.assertTrue(findDuplicates(l, strip=True) == [{0, 1}, {3, 5}]) 227 | self.assertTrue(findDuplicates(l, strip=False) == [{0, 1}]) 228 | 229 | l = ["a", "b", "c"] 230 | self.assertTrue(findDuplicates(l, strip=True) == []) 231 | 232 | 233 | if mini <= 9 <= maxi: 234 | class Test9(unittest.TestCase): 235 | def test1(self): 236 | def check(expected, got): 237 | print("\n") 238 | self.assertTrue(expected is not None) 239 | self.assertTrue(got is not None) 240 | print("Expected: " + str(expected)) 241 | print("Got: " + str(got)) 242 | self.assertTrue(len(expected) == len(got)) 243 | for current in got: 244 | self.assertTrue(current in expected) 245 | print("OK") 246 | print("\n") 247 | 248 | d1 = [{1, 2}, {3, 4}] 249 | d2 = [{2, 4}, {5, 10}] 250 | expected = [{1, 2, 3, 4}, {5, 10}] 251 | got = mergeDuplicates([d1, d2]) 252 | check(expected, got) 253 | 254 | d1 = [{1, 2}, {3, 4}, {5, 6}, {7, 8}, {16, 17}] 255 | d2 = [{11, 12}, {13, 14}, {15, 16}, {10, 8, 11, 3}] 256 | expected = [{1, 2}, {3, 4, 7, 8, 10, 11, 12}, {5, 6}, {13, 14}, {15, 16, 17}] 257 | got = mergeDuplicates([d1, d2]) 258 | check(expected, got) 259 | 260 | d1 = [{1, 2, 3}] 261 | d2 = [{11, 12, 13}] 262 | d3 = [{3, 11}] 263 | d4 = [{15, 16}] 264 | expected = [{1, 2, 3, 11, 12, 13}, {15, 16}] 265 | got = mergeDuplicates([d1, d2, d3, d4]) 266 | check(expected, got) 267 | 268 | d1 = [{1, 2, 3}] 269 | d2 = [{11, 12, 13}] 270 | d3 = [{3, 11}] 271 | d4 = [{15, 16}] 272 | expected = [{1, 2, 3, 11, 12, 13}, {15, 16}] 273 | got = mergeDuplicates([d1, d2, d3, d4]) 274 | check(expected, got) 275 | 276 | 277 | 278 | if mini <= 10 <= maxi: 279 | class Test10(unittest.TestCase): 280 | def test1(self): 281 | gotException = False 282 | try: 283 | ratioSplit(["e"] * 11, [0.8, 0.1, 0.1]) 284 | ratioSplit(["e"] * 12, [0.8, 0.1, 0.1]) 285 | ratioSplit(["e"] * 16, [0.8, 0.1, 0.1]) 286 | except: gotException = True 287 | self.assertTrue(not gotException) 288 | 289 | 290 | l = [1, 2, 3, 4, 5] 291 | self.assertTrue(ratioSplit(l, [0.05, 0.05, 0.9]) == [[1], [2], [3, 4, 5]]) 292 | self.assertTrue(ratioSplit(l, [0.9, 0.05, 0.05]) == [[1, 2, 3], [4], [5]]) 293 | self.assertTrue(ratioSplit(l, [0.4, 0.6]) == [[1, 2], [3, 4, 5]]) 294 | self.assertTrue(ratioSplit(l, [0.6, 0.4]) == [[1, 2, 3], [4, 5]]) 295 | 296 | l = [1, 2, 3, 4] 297 | self.assertTrue(ratioSplit(l, [0.4, 0.2, 0.3, 0.1]) == [[1], [2], [3], [4]]) 298 | self.assertTrue(ratioSplit(l, [0.1, 0.1, 0.8]) == [[1], [2], [3, 4]]) 299 | self.assertTrue(ratioSplit(l, [0.9, 0.1]) == [[1, 2, 3], [4]]) 300 | self.assertTrue(ratioSplit(l, [0.5, 0.5]) == [[1, 2], [3, 4]]) 301 | self.assertTrue(ratioSplit(l, [1.0]) == [[1, 2, 3, 4]]) 302 | self.assertTrue(ratioSplit(l, [0.8, 0.2]) == [[1, 2, 3], [4]]) 303 | 304 | gotException = False 305 | try: 306 | ratioSplit([1], [1.0, 0.2]) 307 | except: gotException = True 308 | self.assertTrue(gotException) 309 | 310 | gotException = False 311 | try: 312 | ratioSplit([1, 2], [0.5, 0.2, 0.3]) 313 | except: gotException = True 314 | self.assertTrue(gotException) 315 | 316 | gotException = False 317 | try: 318 | ratioSplit([1, 2], [0.5, 0.2, 0.3]) 319 | except: gotException = True 320 | self.assertTrue(gotException) 321 | 322 | gotException = False 323 | try: 324 | ratioSplit([1, 2], [0.5, 0.2]) 325 | except: gotException = True 326 | self.assertTrue(gotException) 327 | 328 | l = list(range(10)) 329 | self.assertTrue(ratioSplit(l, [0.1] * 10) == [[i] for i in range(10)]) 330 | 331 | 332 | 333 | 334 | 335 | if __name__ == '__main__': 336 | unittest.main() # Orb execute as Python unit-test in eclipse 337 | 338 | 339 | print("Unit tests done.\n==============") -------------------------------------------------------------------------------- /systemtools/number.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import re 4 | 5 | def parseNumber(text): 6 | """ 7 | Return the first number in the given text for any locale. 8 | TODO we actually don't take into account spaces for only 9 | 3-digited numbers (like "1 000") so, for now, "1 0" is 10. 10 | TODO parse cases like "125,000.1,0.2" (125000.1). 11 | 12 | :example: 13 | >>> parseNumber("a 125,00 €") 14 | 125 15 | >>> parseNumber("100.000,000") 16 | 100000 17 | >>> parseNumber("100 000,000") 18 | 100000 19 | >>> parseNumber("100,000,000") 20 | 100000000 21 | >>> parseNumber("100 000 000") 22 | 100000000 23 | >>> parseNumber("100.001 001") 24 | 100.001 25 | >>> parseNumber("$.3") 26 | 0.3 27 | >>> parseNumber(".003") 28 | 0.003 29 | >>> parseNumber(".003 55") 30 | 0.003 31 | >>> parseNumber("3 005") 32 | 3005 33 | >>> parseNumber("1.190,00 €") 34 | 1190 35 | >>> parseNumber("1190,00 €") 36 | 1190 37 | >>> parseNumber("1,190.00 €") 38 | 1190 39 | >>> parseNumber("$1190.00") 40 | 1190 41 | >>> parseNumber("$1 190.99") 42 | 1190.99 43 | >>> parseNumber("$-1 190.99") 44 | -1190.99 45 | >>> parseNumber("1 000 000.3") 46 | 1000000.3 47 | >>> parseNumber('-151.744122') 48 | -151.744122 49 | >>> parseNumber('-1') 50 | -1 51 | >>> parseNumber("1 0002,1.2") 52 | 10002.1 53 | >>> parseNumber("") 54 | 55 | >>> parseNumber(None) 56 | 57 | >>> parseNumber(1) 58 | 1 59 | >>> parseNumber(1.1) 60 | 1.1 61 | >>> parseNumber("rrr1,.2o") 62 | 1 63 | >>> parseNumber("rrr1rrr") 64 | 1 65 | >>> parseNumber("rrr ,.o") 66 | 67 | """ 68 | try: 69 | # First we return None if we don't have something in the text: 70 | if text is None: 71 | return None 72 | if isinstance(text, int) or isinstance(text, float): 73 | return text 74 | text = text.strip() 75 | if text == "": 76 | return None 77 | # Next we get the first "[0-9,. ]+": 78 | n = re.search("-?[0-9]*([,. ]?[0-9]+)+", text).group(0) 79 | n = n.strip() 80 | if not re.match(".*[0-9]+.*", text): 81 | return None 82 | # Then we cut to keep only 2 symbols: 83 | while " " in n and "," in n and "." in n: 84 | index = max(n.rfind(','), n.rfind(' '), n.rfind('.')) 85 | n = n[0:index] 86 | n = n.strip() 87 | # We count the number of symbols: 88 | symbolsCount = 0 89 | for current in [" ", ",", "."]: 90 | if current in n: 91 | symbolsCount += 1 92 | # If we don't have any symbol, we do nothing: 93 | if symbolsCount == 0: 94 | pass 95 | # With one symbol: 96 | elif symbolsCount == 1: 97 | # If this is a space, we just remove all: 98 | if " " in n: 99 | n = n.replace(" ", "") 100 | # Else we set it as a "." if one occurence, or remove it: 101 | else: 102 | theSymbol = "," if "," in n else "." 103 | if n.count(theSymbol) > 1: 104 | n = n.replace(theSymbol, "") 105 | else: 106 | n = n.replace(theSymbol, ".") 107 | else: 108 | # Now replace symbols so the right symbol is "." and all left are "": 109 | rightSymbolIndex = max(n.rfind(','), n.rfind(' '), n.rfind('.')) 110 | rightSymbol = n[rightSymbolIndex:rightSymbolIndex+1] 111 | if rightSymbol == " ": 112 | return parseNumber(n.replace(" ", "_")) 113 | n = n.replace(rightSymbol, "R") 114 | leftSymbolIndex = max(n.rfind(','), n.rfind(' '), n.rfind('.')) 115 | leftSymbol = n[leftSymbolIndex:leftSymbolIndex+1] 116 | n = n.replace(leftSymbol, "L") 117 | n = n.replace("L", "") 118 | n = n.replace("R", ".") 119 | # And we cast the text to float or int: 120 | n = float(n) 121 | if n.is_integer(): 122 | return int(n) 123 | else: 124 | return n 125 | except: pass 126 | return None 127 | 128 | 129 | def truncateFloat(f, n=2): 130 | '''Truncates/pads a float f to n decimal places without rounding''' 131 | s = '{}'.format(f) 132 | if 'e' in s or 'E' in s: 133 | return float('{0:.{1}f}'.format(f, n)) 134 | i, p, d = s.partition('.') 135 | return float('.'.join([i, (d+'0'*n)[:n]])) 136 | 137 | 138 | 139 | def removeCommasBetweenDigits(text): 140 | """ 141 | :example: 142 | >>> removeCommasBetweenDigits("sfeyv dsf,54dsf ef 6, 6 zdgy 6,919 Photos and 3,3 videos6,") 143 | 'sfeyv dsf,54dsf ef 6, 6 zdgy 6919 Photos and 33 videos6,' 144 | """ 145 | if text is None: 146 | return None 147 | else: 148 | return re.sub(r"([0-9]),([0-9])", "\g<1>\g<2>", text) 149 | 150 | def getAllNumbers(text, removeCommas=False): 151 | if text is None: 152 | return None 153 | if removeCommas: 154 | text = removeCommasBetweenDigits(text) 155 | allNumbers = [] 156 | if len(text) > 0: 157 | # Remove space between digits : 158 | spaceNumberExists = True 159 | while spaceNumberExists: 160 | text = re.sub('(([^.,0-9]|^)[0-9]+) ([0-9])', '\\1\\3', text, flags=re.UNICODE) 161 | if re.search('([^.,0-9]|^)[0-9]+ [0-9]', text) is None: 162 | spaceNumberExists = False 163 | numberRegex = '[-+]?[0-9]+[.,][0-9]+|[0-9]+' 164 | allMatchIter = re.finditer(numberRegex, text) 165 | if allMatchIter is not None: 166 | for current in allMatchIter: 167 | currentFloat = current.group() 168 | currentFloat = re.sub("\s", "", currentFloat) 169 | currentFloat = re.sub(",", ".", currentFloat) 170 | currentFloat = float(currentFloat) 171 | if currentFloat.is_integer(): 172 | allNumbers.append(int(currentFloat)) 173 | else: 174 | allNumbers.append(currentFloat) 175 | return allNumbers 176 | 177 | def removeAllNumbers(text): 178 | if text is None: 179 | return None 180 | if len(text) == 0: 181 | return "" 182 | # Remove space between digits : 183 | spaceNumberExists = True 184 | while spaceNumberExists: 185 | text = re.sub('([0-9]) ([0-9])', '\\1\\2', text, flags=re.UNICODE) 186 | if re.search('[0-9] [0-9]', text) is None: 187 | spaceNumberExists = False 188 | numberRegex = '[-+]?[0-9]+[.,][0-9]+|[0-9]+' 189 | numberExists = True 190 | while numberExists: 191 | text = re.sub(numberRegex, "", text) 192 | if re.search(numberRegex, text) is None: 193 | numberExists = False 194 | 195 | return text.strip() 196 | 197 | def getFirstNumber(text, *args, **kwargs): 198 | result = getAllNumbers(text, *args, **kwargs) 199 | if result is not None and len(result) > 0: 200 | return result[0] 201 | return None 202 | 203 | def representsFloat(text): 204 | """ 205 | This function return True if the given param (string or float) represents a float 206 | 207 | :Example: 208 | >>> representsFloat("1.0") 209 | True 210 | >>> representsFloat("1") 211 | False 212 | >>> representsFloat("a") 213 | False 214 | >>> representsFloat(".0") 215 | False 216 | >>> representsFloat("0.") 217 | False 218 | >>> representsFloat("0.000001") 219 | True 220 | >>> representsFloat("00000.000001") 221 | True 222 | >>> representsFloat("0000a0.000001") 223 | False 224 | """ 225 | if isinstance(text, float): 226 | return True 227 | elif text is None: 228 | return False 229 | elif isinstance(text, str): 230 | if len(text) < 3: 231 | return False 232 | text = text.strip() 233 | return re.search("^[0-9]{1,}\.[0-9]{1,}$", text) is not None 234 | else: 235 | return False 236 | 237 | 238 | 239 | def representsInt(s, acceptRoundedFloats=False): 240 | """ 241 | This function return True if the given param (string or float) represents a int 242 | 243 | :Example: 244 | >>> representsInt(1) 245 | True 246 | >>> representsInt("1") 247 | True 248 | >>> representsInt("a") 249 | False 250 | >>> representsInt("1.1") 251 | False 252 | >>> representsInt(1.1) 253 | False 254 | >>> representsInt(42.0, acceptRoundedFloats=True) 255 | True 256 | >>> representsInt("42.0", acceptRoundedFloats=True) 257 | True 258 | """ 259 | 260 | if isinstance(s, float): 261 | if acceptRoundedFloats: 262 | return s.is_integer() 263 | else: 264 | if acceptRoundedFloats: 265 | try: 266 | s = float(s) 267 | return representsInt(s, acceptRoundedFloats=acceptRoundedFloats) 268 | except ValueError: 269 | return False 270 | else: 271 | try: 272 | int(s) 273 | return True 274 | except ValueError: 275 | return False 276 | return False 277 | 278 | 279 | def floatAsReadable(f): 280 | """ 281 | source https://stackoverflow.com/questions/8345795/force-python-to-not-output-a-float-in-standard-form-scientific-notation-expo 282 | """ 283 | _ftod_r = re.compile(br'^(-?)([0-9]*)(?:\.([0-9]*))?(?:[eE]([+-][0-9]+))?$') 284 | """Print a floating-point number in the format expected by PDF: 285 | as short as possible, no exponential notation.""" 286 | s = bytes(str(f), 'ascii') 287 | m = _ftod_r.match(s) 288 | if not m: 289 | raise RuntimeError("unexpected floating point number format: {!a}" 290 | .format(s)) 291 | sign = m.group(1) 292 | intpart = m.group(2) 293 | fractpart = m.group(3) 294 | exponent = m.group(4) 295 | if ((intpart is None or intpart == b'') and 296 | (fractpart is None or fractpart == b'')): 297 | raise RuntimeError("unexpected floating point number format: {!a}" 298 | .format(s)) 299 | 300 | # strip leading and trailing zeros 301 | if intpart is None: intpart = b'' 302 | else: intpart = intpart.lstrip(b'0') 303 | if fractpart is None: fractpart = b'' 304 | else: fractpart = fractpart.rstrip(b'0') 305 | 306 | result = None 307 | 308 | if intpart == b'' and fractpart == b'': 309 | # zero or negative zero; negative zero is not useful in PDF 310 | # we can ignore the exponent in this case 311 | result = b'0' 312 | 313 | # convert exponent to a decimal point shift 314 | elif exponent is not None: 315 | exponent = int(exponent) 316 | exponent += len(intpart) 317 | digits = intpart + fractpart 318 | if exponent <= 0: 319 | result = sign + b'.' + b'0'*(-exponent) + digits 320 | elif exponent >= len(digits): 321 | result = sign + digits + b'0'*(exponent - len(digits)) 322 | else: 323 | result = sign + digits[:exponent] + b'.' + digits[exponent:] 324 | 325 | # no exponent, just reassemble the number 326 | elif fractpart == b'': 327 | result = sign + intpart # no need for trailing dot 328 | else: 329 | result = sign + intpart + b'.' + fractpart 330 | 331 | result = result.decode("utf-8") 332 | if result.startswith("."): 333 | result = "0" + result 334 | return result 335 | 336 | 337 | 338 | def digitalizeIntegers(text, totalDigits=100): 339 | if text is None or not isinstance(text, str) or text == "": 340 | return text 341 | result = str(text) 342 | toEdit = [] 343 | for current in re.finditer("[0-9]+", text): 344 | theInt = current.group(0) 345 | start = current.start(0) 346 | end = current.end(0) 347 | remainingDigits = totalDigits - len(theInt) 348 | digitalizedInt = "0" * remainingDigits + theInt 349 | toEdit.append((digitalizedInt, start, end)) 350 | for digitalizedInt, start, end in reversed(toEdit): 351 | # print(digitalizedInt, start, end) 352 | result = result[:start] + digitalizedInt + result[end:] 353 | return result 354 | 355 | def main(): 356 | allTexts = \ 357 | [ 358 | "ttt1ttt3t", 359 | "zzz23.32zzz8", 360 | "3.0z", 361 | "aaaaa", 362 | "bb", 363 | None, 364 | "1111111111111111111111111111111111111111111", 365 | "5", 366 | "0", 367 | ] 368 | for current in allTexts: 369 | print(current) 370 | print(digitalizeIntegers(current)) 371 | print() 372 | 373 | if __name__ == '__main__': 374 | # print(list(range(1, -1, -1))) 375 | main() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # SystemTools 3 | 4 | This project gathers some useful Python functions and classes. We organized them in different modules: 5 | 6 | * **systemtools.printer** 7 | * **systemtools.duration** 8 | * **systemtools.logger** 9 | * **systemtools.location** 10 | * **systemtools.basics** 11 | * **systemtools.number** 12 | * **systemtools.file** 13 | * **systemtools.system** 14 | 15 | To install all: 16 | 17 | pip install systools 18 | 19 | 20 | Optionnal requirement: 21 | 22 | github.com/Pithikos/winlaunch.git 23 | 24 | ## systemtools.printer 25 | 26 | This module provide function that beautiful print variables. 27 | 28 | >>> from systemtools.printer import * 29 | >>> bp(['Some text lorem ipsum dolor sit amet', {'a': 'This is some text that is short.', 'b': 'This is a longer text lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit amet.'}, 'Here an other text', 8, 9, [2, 3, 4, 5], 10, {1, 2, 3}]) 30 | [ 31 | Some text lorem ipsum dolor sit amet, 32 | { 33 | a: This is some text that is short., 34 | b: This is a longer text lorem ipsum dolor sit amet lorem ipsum dolor sit amet lorem ipsum dolor sit am 35 | }, 36 | ..., 37 | 10, 38 | { 1, 2, 3 } 39 | ] 40 | 41 | Use the level as second argument (from 0 to 5) to set the verbosity of the print. 42 | 43 | ## systemtools.duration 44 | 45 | This module provide some useful class to handle time. 46 | 47 | >>> from systemtools.duration import * 48 | 49 | ### TicToc 50 | 51 | This class allow an easy print of computation time in your scripts: 52 | 53 | >>> tt = TicToc() 54 | >>> tt.tic() # Start the timer 55 | >>> 56 | >>> tt.tic() # Print the time duration (human readable) since the previous tic() 57 | tic: 1s 58 | >>> 59 | >>> tt.tic() 60 | tic: 1s 61 | >>> tt.toc() # Print the total duration 62 | toc total duration: 2s 63 | 64 | You can give `msg` parameter to add a message to the printed duration. You can also choose to do not display anything using `display=False`. 65 | 66 | Both `tic` and `toc` methods return the time spent in seconds. 67 | 68 | ### ProgressBar 69 | 70 | A light alternative to [tqdm](https://github.com/tqdm/tqdm). Just wrap your iterables with the `pb` funct: 71 | 72 | ```python 73 | for i in pb(range(iterationAmount)): 74 | time.sleep(0.01) 75 | ``` 76 | 77 | This will display: 78 | 79 | 0% [ ] 80 | 20% [==== ] (1.6s left) 81 | 40% [======== ] (1.214s left) 82 | 60% [============ ] (0.813s left) 83 | 80% [================ ] (0.404s left) 84 | 100% [====================] (total duration: 2.03s, mean duration: 0.01s) 85 | 86 | By default, `pb` will **not** display more than 10 messages to do not display too much progress informations in the case you used the `nohup` command, or used a `Logger` for example. 87 | 88 | `pb` take same parameters as the `ProgressBar` class init parameters. You can use the class directly to handle your progress bar by hand giving an iteration amount and by calling the `tic()` method at each iteration: 89 | 90 | ```python 91 | iterationAmount = 200 92 | pb = ProgressBar(iterationAmount) 93 | for i in range(iterationAmount): 94 | time.sleep(0.01) 95 | pb.tic(i) # Give a message to the `tic` method to display informations about the current iteration 96 | ``` 97 | 98 | If you work on a terminal, it will automatically display informations more frenquently and replace the current line. 99 | 100 | Init parameters are ([see the code for more information](https://github.com/hayj/SystemTools/blob/master/systemtools/duration.py#L179)): 101 | 102 | * **message**: will display this message at each `tic()` 103 | * **printRatio**: display a message at each `printRatio * iterationAmount` times you call `tic()`. Default is 0.1, meaning it will display 10%, 20%... 104 | 105 | `tic()` parameters are: 106 | 107 | * **extraMessage**: use this message if you want to display informations about the current iteration. 108 | 109 | *Tested in Python 3 on Ubuntu.* 110 | 111 | ### Timer 112 | 113 | This class call a function each n seconds: 114 | 115 | >>> timer = Timer(myFunct, 5) 116 | >>> timer.start() 117 | 118 | 119 | You can stop it using: 120 | 121 | >>> timer.stop() 122 | 123 | Set `sleepFirst=True` if you don't want to call your funct at the startup of the timer. 124 | 125 | ## systemtools.logger 126 | 127 | A Logger class is a wrapper over `logging`. 128 | 129 | >>> from systemtools.logger import * 130 | >>> logger = Logger("test.log") # Give a file path 131 | >>> logger.info("a") # Print infos 132 | >>> logger.error("b") # Print errors... 133 | 134 | If you created a class which contains `logger` and `verbose` like this one: 135 | 136 | >>> class LoggerTest: 137 | ... def __init__(self, logger=None, verbose=True): 138 | ... self.logger = logger 139 | ... self.verbose = verbose 140 | 141 | And use functions `log`, `logError`... this way in a method of your class: 142 | 143 | ... log("Initialized....", self) 144 | 145 | So the log function will automaticllay check if verbose is True, and if there is no `logger`, it will simply print your message. 146 | 147 | You can also use `logException` this way: 148 | 149 | ... logException(e, self) # You can give message (string) and location (string) parameters 150 | 151 | You can also give a `Logger` instead of a class instance: 152 | 153 | >>> log("a", logger) 154 | >>> logException(e, logger, verbose=myVerbose) 155 | >>> ... 156 | 157 | ## systemtools.location 158 | 159 | This module gathers some useful functions on file system location. 160 | 161 | >>> from systemtools.location import * 162 | 163 | * **sortedGlob(regex, caseSensitive=True, sortBy=GlobSortEnum.NAME, reverse=False)**: This function works the same as glob.glob but return an ordered list of files path. glob.glob return (by default) a ordered list which can change across OS or executions and it is prone to errors in your python script. You can use different orders via sortBy: GlobSortEnum. the last one is the same as name but take into account numbers (e.g. test1.txt < test10.txt). 164 | * **homeDir()** : Return the path of your home dir. 165 | * **tmpDir(_file_=None, subDir=None)**: Return the path of the tmp dir, If you give `__file__` in first param, the tmp dir will be "tmp" in the current directory, else it will be ~/tmp. You can set `subDir` in parameters. 166 | * **execDir(_file_=None)**: Get the current directory, it is better to give `__file__` in parameter to be sure to get the dir of the current Python script. 167 | * **isDir(path)**: Return True is the given path is a directory. 168 | * **isFile(path)**: Return True is the given path is a file. 169 | * **decomposePath(path)**: Return a tuple (dir, filename, ext, filenameAndExt) of a path. 170 | 171 | You can set the default tmp directory: 172 | 173 | ```python 174 | from systemtools import config as systConf 175 | systConf.defaultTmpDir = "/your/tmp/directory" 176 | ``` 177 | 178 | ## systemtools.basics 179 | 180 | This module gathers some useful basics functions. 181 | 182 | >>> from systemtools.basics import * 183 | 184 | * **listSubstract(a, b)**: Substract all `b` items from `a`. 185 | * **convertDate(readableDate=None, dateFormat=DATE_FORMAT.datetime)**: Convert a readable date (wrote by a human) in a date format you chose. Warning : utc shift may appear. DATE_FORMAT enum contains "datetimeString datetime timestamp arrow arrowString humanize". 186 | * **mergeDicts(dict1, ...)**: shallow copy of all dict and merge into a new dict 187 | * **reduceDictStr**: See the code for parameters. Reduce all strings of a dict in order to print it. 188 | * **stripAccents(text)**: Remove all accents of a string. 189 | * **printLTS(l)**: Pretty print a list or a dict. Use `listToStr` internally. 190 | * **listToStr(l)**: Convert a list or a dict to a pretty string. 191 | * **floatAsReadable**: Convert a float to a readable string without "e-X". 192 | * **sortByKey(d)**: Sort a dict by the key. Return an `OrderedDict`. 193 | * **sortBy(l, desc=False, index=1)**: Sort a list of tuple (or an itemized dict) by the specific index given in parameters. 194 | * **chunks(l, n)**: return a list of lists (of len n) from `l`. You can also use `chunksYielder`. 195 | * **split(l, n)**: split a list in n parts. 196 | * **splitMaxSized(l, batchMaxSize)**: Split a list in multiple parts in such a way that each part has a max size of batchMaxSize. 197 | * **normalize(l)**: Normalize (between 0.0 and 1.0) all elements of a list according to the sum of all elements. 198 | * **getRandomInt(a=None, b=None, seed=None, count=1)**: Return a random int between `a` and `b`. 199 | * **getRandomFloat(min=0.0, max=1.0, decimalMax=2)**: Return a random float between `min` and `max`. 200 | * **getRandomStr(digitCount=10, withTimestamp=True)**: Return a random string with a timestamp if enabled. 201 | * **getRandomName(addInt=True, maxInt=100)**: Return a random name with a random int. 202 | * **Random class**: This class is useful when you want to seed random values and reset it after the class usage. See the code for more informations. 203 | * **dictContains(d, key)**: Equivalent to `key in d and d[key] is not None`. 204 | * **intersection(lists)**: Return the intersection of all lists. 205 | * **reduceStr**: Reduce a str, you can set booleans removeNumbers, toLowerCase, removeAccents and removePunct. 206 | * **varname(p)**: Return the name of p from the Python script. 207 | * **stripAllLines(text, removeBlank=True)**: Return the text but strip all lines. 208 | * **byteToStr(b)**: Convert bytes to str. 209 | * **getDictSubElement(theDict, keys)**: This function browse the dict as a tree and return the value in the path defined by keys which is a list of dict keys. It return None if it doesn't find anything. Example: `getDictSubElement({'a': {'b': 1}}, ['a', 'b'])` return `1`. 210 | * **objectAsKey(o)**: Convert any object to a key, if if instead call `str(o)` or `repr(o)`, the string can change over executions of your script due to the unordered nature of dictionnaries and sets. 211 | * **reducedLTS(o, amount=25)**: Same as `lts(o)` but keep only `amount` elements at the head and the tail of the object if it is a list. 212 | * **reduceBlank(text, keepNewLines=False) (aslias stripAll, trimAll)**: Strip a string and reduce all blank space to a unique space. If you set keepNewLines as True, it will keep a unique '\n' at each blank space which contains a '\n' or a '\r'. 213 | * **linearScore(x, x1=0.0, x2=1.0, y1=0.0, y2=1.0, stayBetween0And1=True)**: Give you a score f(x) defined by the linear function line (x1, y1) (x2, y2). 214 | * **camelCaseToUnderscoreCase(name)**: Convert a string which is formatted as the camelCase norm to the underscore_case norm. 215 | * **camelCaseToUnderscoreCaseDict(theDict)**: Turn each key of the dict according to `camelCaseToUnderscoreCase`. 216 | * **tuplesToDict(tupleList)**: Convert a list of tuples to a dict in such a way that the first element of each tuple will be the key. 217 | * **findDuplicates(texts, strip=True)**: Return a list a duplicates (indexes of texts in th list). 218 | * **intByteSize(n)**: Return the size of an integer in bytes. 219 | 220 | ## systemtools.number 221 | 222 | This module gathers some useful basics functions on number parsing. 223 | 224 | >>> from systemtools.number import * 225 | 226 | * **parseNumber(text)**: Return the first number in the given text for any locale. 227 | * **getAllNumbers(text, removeCommas=False)**: Return all numbers in a string. You can also use `getFirstNumber`. 228 | * **getFirstNumber(text)**: Get the first numbers of a string. 229 | * **removeAllNumbers(text)**: Remove all numbers from a string. 230 | * **truncateFloat(f, n)**: Truncates/pads a float f to n decimal places without rounding. 231 | 232 | ## systemtools.file 233 | 234 | This module gathers some useful functions on file and directories management. 235 | 236 | >>> from systemtools.file import * 237 | 238 | * **getLastModifiedTimeSpent(path, timeSpentUnit=TIMESPENT_UNIT.HOURS)**: Return the time spent after the last modified event on a path (file or directory). 239 | * **strToFilename(text)**: Convert a string in a filename (storable on the disk). So it will remove all non permitted chars. 240 | * **mkdir(path)**: Create a directory if it doesn't already exist. 241 | * **globRemove(globPattern)**: Remove file according to a glob pattern similar to the glob lib. 242 | * **removeFile(path)**: Remove a file or a list of files. 243 | * **fileToStr(path)**: Load a file and return the string in. 244 | * **fileToStrList**: Load a file and return a list of strings. You can set `strip` as `False` to don't strip all lines, `skipBlank` as `False` to keep blank lines, you can choose your comment start indicator using `commentStart` (default is "###"). 245 | * **strToFile(text, path)**: Store a string in a file. 246 | * **strToTmpFile(text, name=None, ext="", addRandomStr=False)**: Store a string to a tmp file (using `tmpDir` function). Example: strToTmpFile("a", "test.txt"). 247 | * **strListToTmpFile**: Use `strToTmpFile` but for a list of strings which is concatened. 248 | * **normalizeNumericalFilePaths(globRegex)**: This function get a glob path and rename all "file1.json", "file2.json"... "file20.json" to "file01.json", "file02.json"... "file20.json" to better sort the folder by file names. 249 | * **encryptFile(path, key, text=None, remove=True)**: This function encrypt a file, if you give text in `text` parameter, the function will create the file. Return True if all is ok. You need to install 7zip using `sudo apt-get install p7zip-full` on Linux. Set remove as `False` if you don't want to remove the decrypted file. 250 | * **decryptFile(path, key, remove=True)**: This function decrypt a file and return the text. Set remove as `False` if you don't want to remove the encrypted file. 251 | 252 | ## systemtools.system 253 | 254 | This module gathers some useful functions on the OS management. 255 | 256 | >>> from systemtools.system import * 257 | 258 | * **getUsedPorts()**: Return all busy ports on your machine (works on Linux using netstat). 259 | * **getUser()**: Equivalent to `getpass.getuser()` 260 | * **setCallTimeout(timeout) and resetCallTimeout()**: Use `setCallTimeout` to set a timeout before calling a function (so you can catch an Exception if the function is too long), then reset the timeout. 261 | * **getRAMTotal()**: Return the amount of RAM in Go 262 | * **cpuCount()**: Equivalent to `multiprocessing.cpu_count()` 263 | * **isHostname(h)**: Return `True` if the hostname of the current computer starts with `h`. 264 | * **getHostname()**: Equivalent to `socket.gethostname()` 265 | * **randomSleep(min=0.1, max=None)**: Sleep between min and max. If max is None: max = min + 0.2 * min. 266 | * **getMemoryPercent()**: Return the % of memory usage. 267 | 268 | -------------------------------------------------------------------------------- /systemtools/duration.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # pew in st-venv python /home/hayj/Workspace/Python/Utils/SystemTools/systemtools/duration.py 4 | 5 | import time 6 | import re 7 | from systemtools.logger import * 8 | from systemtools.basics import * 9 | from threading import Thread 10 | import os 11 | import sys 12 | from enum import Enum 13 | import signal 14 | from multiprocessing import Queue 15 | from threading import Thread 16 | 17 | class Timer: 18 | def __init__(self, callback, interval, *args, sleepFirst=False, sleepCount=1000, logger=None, verbose=True, **kwargs): 19 | """ 20 | interval in seconds 21 | """ 22 | self.logger = logger 23 | self.verbose = verbose 24 | self.sleepCount = sleepCount 25 | self.interval = interval 26 | self.callback = callback 27 | self.stopped = True 28 | self.firstExec = True 29 | self.sleepFirst = sleepFirst 30 | self.mainThread = None 31 | self.setArgs(*args, **kwargs) 32 | 33 | def setArgs(self, *args, **kwargs): 34 | self.args = args 35 | self.kwargs = kwargs 36 | 37 | def isRunning(self): 38 | return not self.stopped 39 | 40 | def sleep(self): 41 | sleepPart = self.interval / self.sleepCount 42 | for i in range(self.sleepCount): 43 | if self.isRunning(): 44 | time.sleep(sleepPart) 45 | 46 | def run(self): 47 | self.stopped = False 48 | while not self.stopped: 49 | if self.firstExec and self.sleepFirst: 50 | self.sleep() 51 | if self.isRunning(): 52 | try: 53 | self.callback(*self.args, **self.kwargs) 54 | except Exception as e: 55 | logException(e, self) 56 | self.sleep() 57 | self.firstExec = False 58 | 59 | def start(self): 60 | if self.stopped: 61 | self.firstExec = True 62 | self.mainThread = Thread(target=self.run) 63 | self.mainThread.start() 64 | 65 | def stop(self): 66 | self.stopped = True 67 | 68 | 69 | 70 | class TicToc(): 71 | """ 72 | This class provide 2 methods to print time during an execution 73 | """ 74 | def __init__(self, verbose=True, logger=None, marker="-->", msgSeparator=" | message: ", maxDecimal=2, doWarnFreeRAM=False): 75 | self.verbose = verbose 76 | self.logger = logger 77 | self.startTime = None 78 | self.previousTime = None 79 | self.marker = marker 80 | self.msgSeparator = msgSeparator 81 | self.maxDecimal = maxDecimal 82 | self.doWarnFreeRAM = doWarnFreeRAM 83 | 84 | def setMaxDecimal(self, maxDecimal): 85 | self.maxDecimal = maxDecimal; 86 | 87 | def tic(self, msg=None, display=True): 88 | """ 89 | This method start the timer and print it, or print the time between the previous tic() 90 | and the current tic(). You can print a message by giving it in parameters. 91 | It's the local duration. 92 | """ 93 | if msg is None: 94 | msg = ""; 95 | else: 96 | msg = self.msgSeparator + msg; 97 | if self.startTime is None: 98 | self.startTime = time.time(); 99 | self.previousTime = self.startTime; 100 | if display: 101 | self.p(self.marker + " tictoc starts..." + msg); 102 | return -1; 103 | else: 104 | currentTime = time.time(); 105 | diffTime = currentTime - self.previousTime; 106 | diffTime = float(float(int(diffTime * (10**self.maxDecimal))) / float((10**self.maxDecimal))); 107 | if display: 108 | self.p(self.marker + " tic: " + secondsToHumanReadableDuration(diffTime) + msg); # time duration from the previous tic() 109 | self.previousTime = currentTime; 110 | return diffTime; 111 | 112 | def toc(self, msg=None, display=True): 113 | """ 114 | This method print the elapsed time from the first tic(). 115 | You can print a message by giving it in parameters. 116 | It's the total duration. 117 | """ 118 | if self.startTime is not None: 119 | if msg is None: 120 | msg = ""; 121 | else: 122 | msg = self.msgSeparator + msg; 123 | currentTime = time.time(); 124 | diffTime = currentTime - self.startTime; 125 | diffTime = float(float(int(diffTime * (10**self.maxDecimal))) / float((10**self.maxDecimal))); 126 | if display: 127 | self.p(self.marker + " toc total duration: " + secondsToHumanReadableDuration(diffTime) + msg); 128 | return diffTime; 129 | return -1; 130 | 131 | def p(self, text): 132 | log(text, self) 133 | # if self.logger is not None: 134 | # self.logger.p(text) 135 | # else: 136 | # print(text) 137 | 138 | def secondsToHumanReadable(*args, **kwargs): 139 | return secondsToHumanReadableDuration(*args, **kwargs) 140 | def secondsToHumanReadableDuration(seconds): 141 | """ 142 | :example: 143 | >>> secondsToHumanReadableDuration(0.1) 144 | '0.1s' 145 | >>> secondsToHumanReadableDuration(10) 146 | '10.0s' 147 | >>> secondsToHumanReadableDuration(10.2) 148 | '10.2s' 149 | >>> secondsToHumanReadableDuration(3600) 150 | '1h 0m 0.0s' 151 | >>> secondsToHumanReadableDuration(7210) 152 | '2h 0m 10.0s' 153 | >>> secondsToHumanReadableDuration(7270) 154 | '2h 1m 10.0s' 155 | """ 156 | m, s = divmod(seconds, 60.0) 157 | h, m = divmod(m, 60.0) 158 | h = int(h) 159 | m = int(m) 160 | result = "" 161 | if h != 0: 162 | result += str(h) + "h " 163 | result += str(m) + "m " 164 | elif m != 0: 165 | result += str(m) + "m " 166 | result += floatAsReadable(truncateFloat(s, 3)) + "s" 167 | return result 168 | 169 | 170 | 171 | 172 | 173 | OUTPUT_TYPE = Enum("OUTPUT_TYPE", "standard subl nohup logger") 174 | def getOutputType(logger=None): 175 | if logger is not None: 176 | return OUTPUT_TYPE.logger 177 | if signal.getsignal(signal.SIGHUP) == signal.SIG_DFL: 178 | try: 179 | size = os.get_terminal_size() 180 | except: 181 | return OUTPUT_TYPE.subl 182 | return OUTPUT_TYPE.standard 183 | else: 184 | return OUTPUT_TYPE.nohup 185 | 186 | def canCleanOutput(*args, **kwargs): 187 | return getOutputType(*args, **kwargs) == OUTPUT_TYPE.standard 188 | 189 | class ProgressBar: 190 | def __init__\ 191 | ( 192 | self, 193 | iterationAmount, 194 | message=None, 195 | printRatio=None, # Auto 196 | logger=None, 197 | verbose=True, 198 | defaultPrintRatio=0.1, 199 | progressSymbol="=", 200 | progressSymbolAmount=20, 201 | printProgressBar=True, 202 | printFloatingPoint=None, # Auto 203 | allowTerminalClean=True, 204 | extraMessageOnTheLeft=True, 205 | minIterationCountForRemainingMessage=100, 206 | minDoneRatioForRemainingMessage=0.05, 207 | ): 208 | self.queueThread = None 209 | self.iterQueue = False 210 | self.queue = None 211 | self.minDoneRatioForRemainingMessage = minDoneRatioForRemainingMessage 212 | self.minIterationCountForRemainingMessage = minIterationCountForRemainingMessage 213 | self.extraMessageOnTheLeft = extraMessageOnTheLeft 214 | self.allowTerminalClean = allowTerminalClean 215 | self.printFloatingPoint = printFloatingPoint 216 | self.progressSymbolAmount = progressSymbolAmount 217 | self.printProgressBar = printProgressBar 218 | self.progressSymbol = progressSymbol 219 | self.iterationAmount = iterationAmount 220 | self.message = message 221 | self.printRatio = printRatio 222 | self.logger = logger 223 | self.verbose = verbose 224 | self.outputType = getOutputType(logger=self.logger) 225 | if not self.allowTerminalClean: 226 | self.canCleanOutput = False 227 | else: 228 | self.canCleanOutput = canCleanOutput(logger=self.logger) 229 | if self.printRatio is None: 230 | if self.outputType == OUTPUT_TYPE.standard: 231 | self.printRatio = 0.0001 232 | else: 233 | self.printRatio = defaultPrintRatio 234 | if self.printFloatingPoint is None: 235 | if self.canCleanOutput or self.printRatio < 0.01: 236 | self.printFloatingPoint = True 237 | else: 238 | self.printFloatingPoint = False 239 | self.tt = TicToc(verbose=False) 240 | self.tt.tic() 241 | self.currentIteration = 0 242 | if self.iterationAmount == 0: 243 | self.toc() 244 | if self.iterationAmount < 200: 245 | self.printFloatingPoint = False 246 | 247 | def tic(self, extraMessage=None, amount=1): 248 | duration = self.tt.tic() 249 | totalDuration = self.tt.toc() 250 | self.currentIteration += amount 251 | if self.iterationAmount == 0: 252 | logWarning("The iterationAmount is 0.", self) 253 | return duration 254 | if self.currentIteration == self.iterationAmount: 255 | self.toc(extraMessage=extraMessage) 256 | return duration 257 | if self.iterationAmount == 0: 258 | doneRatio = 1.0 259 | else: 260 | doneRatio = self.currentIteration / self.iterationAmount 261 | theModulo = int(self.printRatio * self.iterationAmount) 262 | if theModulo == 0: 263 | theModulo = 1 264 | hasToDisplay = self.currentIteration == 1 or self.currentIteration % theModulo == 0 265 | if hasToDisplay: 266 | text = "" 267 | if self.message is not None: 268 | text += self.message + " " 269 | if not self.printFloatingPoint: 270 | percent = math.floor(doneRatio * 100) 271 | aSpace = "" 272 | if percent < 10: 273 | aSpace = " " 274 | percent = " " + aSpace + str(percent) 275 | else: 276 | percent = str(int(doneRatio * 100 * 100 + 100000)) 277 | percent = percent[2:] 278 | if percent.startswith("0"): 279 | percent = " " + percent[1:] 280 | percent = percent[:2] + "." + percent[2:] 281 | 282 | # percent = str(int((truncateFloat(doneRatio * 100, self.floatingPointAmount) + 100000) * 100)) 283 | # percent = percent[1:] 284 | # if len(percent) == 5: 285 | # percent = percent[:2] + "." + percent[2:] 286 | # else: 287 | # percent = " " + percent[:1] + "." + percent[1:] 288 | text += str(percent) + "%" 289 | if self.printProgressBar: 290 | nbSymbols = int(doneRatio * self.progressSymbolAmount) 291 | symbols = self.progressSymbol * nbSymbols + " " * (self.progressSymbolAmount - nbSymbols) 292 | text += " [" + symbols + "]" 293 | if self.extraMessageOnTheLeft: 294 | if extraMessage is not None: 295 | text += " " + str(extraMessage) 296 | if self.currentIteration >= self.minIterationCountForRemainingMessage or doneRatio >= self.minDoneRatioForRemainingMessage: 297 | remainingSecs = (totalDuration / doneRatio) - totalDuration 298 | text += " (" + secondsToHumanReadableDuration(remainingSecs) + " left)" 299 | if not self.extraMessageOnTheLeft: 300 | if extraMessage is not None: 301 | text += " " + str(extraMessage) 302 | if self.canCleanOutput and self.verbose: 303 | print(text, end="\r") 304 | else: 305 | log(text, self) 306 | return duration 307 | 308 | def toc(self, extraMessage=None): 309 | self.currentIteration = self.iterationAmount 310 | totalDuration = self.tt.toc() 311 | iterationAmountForDivision = 1 312 | if self.iterationAmount != 0: 313 | iterationAmountForDivision = self.iterationAmount 314 | meanDurationText = secondsToHumanReadableDuration(totalDuration / iterationAmountForDivision) 315 | totalDurationText = secondsToHumanReadableDuration(totalDuration) 316 | text = "" 317 | if self.message is not None: 318 | text += self.message + " " 319 | if self.printFloatingPoint: 320 | text += " 100%" 321 | else: 322 | text += "100%" 323 | if self.printProgressBar: 324 | symbols = self.progressSymbol * self.progressSymbolAmount 325 | text += " [" + symbols + "]" 326 | if extraMessage is not None: 327 | text += " " + str(extraMessage) 328 | text += " (total duration: " + totalDurationText + ", mean duration: " + meanDurationText + ")" 329 | if self.canCleanOutput and self.verbose: 330 | print(text, end="\r") 331 | print() 332 | else: 333 | log(text, self) 334 | return totalDuration 335 | 336 | def __queueRun(self): 337 | while self.iterQueue: 338 | try: 339 | self.queue.get(timeout=0.5) 340 | self.tic() 341 | except: 342 | pass 343 | 344 | def getQueue(self): 345 | return self.queue 346 | 347 | def startQueue(self): 348 | """ 349 | This function will start a Thread which will look in the dedicated 350 | queue. Use the returned queue to put None and this Thread will tic() 351 | the pbar for you. Works in multiprocessing. 352 | See systemtools.test.queuetest for full example. 353 | """ 354 | self.iterQueue = True 355 | self.queue = Queue() 356 | self.queueThread = Thread(target=self.__queueRun) 357 | self.queueThread.start() 358 | return self.queue 359 | 360 | def stopQueue(self): 361 | 362 | self.iterQueue = False 363 | 364 | 365 | 366 | def pb(items, iterationAmount=None, **kwargs): 367 | if iterationAmount is None: 368 | try: 369 | iterationAmount = len(items) 370 | except: pass 371 | if iterationAmount is None: 372 | raise Exception("Length of items not found.") 373 | p = ProgressBar(iterationAmount, **kwargs) 374 | for current in items: 375 | yield current 376 | p.tic() 377 | 378 | 379 | 380 | 381 | def test1(): 382 | iterationAmount = 30000 383 | pb = ProgressBar(iterationAmount, "test", printRatio=0.0001) 384 | for i in range(iterationAmount): 385 | pb.tic(i) 386 | time.sleep(0.00001) 387 | 388 | 389 | def test2(): 390 | iterationAmount = 20 391 | pb = ProgressBar(iterationAmount, "test", printRatio=0.0001) 392 | for i in range(iterationAmount): 393 | pb.tic(i) 394 | time.sleep(0.1) 395 | 396 | 397 | def test3(): 398 | iterationAmount = 20 399 | pb = ProgressBar(iterationAmount, "test", printRatio=0.5) 400 | for i in range(iterationAmount): 401 | pb.tic(i) 402 | time.sleep(0.1) 403 | 404 | def test4(): 405 | iterationAmount = 200 406 | pb = ProgressBar(iterationAmount) 407 | for i in range(iterationAmount): 408 | time.sleep(0.01) 409 | pb.tic() 410 | 411 | # from tqdm import tqdm 412 | 413 | # def test5(): 414 | # iterationAmount = 20000 415 | # # pb = ProgressBar(iterationAmount) 416 | # for i in tqdm(range(iterationAmount)): 417 | # time.sleep(0.01) 418 | # # pb.tic() 419 | 420 | # def test6(): 421 | # iterationAmount = 20000000 422 | # # pb = ProgressBar(iterationAmount) 423 | # for i in tqdm(range(iterationAmount)): 424 | # time.sleep(0.0001) 425 | # # pb.tic() 426 | 427 | def test8(): 428 | for i in pb(range(200)): 429 | time.sleep(0.01) 430 | 431 | 432 | 433 | if __name__ == '__main__': 434 | test1() 435 | test2() 436 | test3() 437 | test4() 438 | # test5() 439 | # test6() 440 | test8() -------------------------------------------------------------------------------- /systemtools/system.py: -------------------------------------------------------------------------------- 1 | # pew in st-venv python ~/Workspace/Python/Utils/SystemTools/systemtools/system.py 2 | 3 | import subprocess 4 | import time 5 | import psutil 6 | import inspect 7 | import os 8 | import glob 9 | import sys 10 | import re 11 | import random 12 | import string 13 | import collections 14 | import socket 15 | from random import randint 16 | from operator import itemgetter 17 | import multiprocessing 18 | import time 19 | import signal 20 | import getpass 21 | from systemtools.basics import * 22 | from systemtools.file import * 23 | from systemtools.number import * 24 | from psutil import virtual_memory 25 | import traceback 26 | import requests 27 | 28 | def myIp(driver=None, ipUrl="https://api.ipify.org?format=json"): 29 | try: 30 | if driver is None: 31 | data = requests.get(ipUrl).text 32 | else: 33 | data = driver.get(ipUrl).page_source 34 | return re.search("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}", data).group(0) 35 | except: 36 | return None 37 | 38 | def sshStats\ 39 | ( 40 | address, 41 | user=None, 42 | timeout=6, 43 | successToken='connection_success', 44 | ): 45 | """ 46 | This function return stats on a server. 47 | https://askubuntu.com/questions/9487/whats-the-difference-between-load-average-and-cpu-load 48 | """ 49 | if user is None: 50 | user = getUser() 51 | results = bash('timeout ' + str(timeout+1) + ' ssh -o BatchMode=yes -o ConnectTimeout=' + str(timeout) + ' ' + user + '@' + address + ' "echo ' + successToken + ' ; uptime ; nproc ; grep MemTotal /proc/meminfo ; grep MemAvailable /proc/meminfo"', doPrint=False) 52 | if successToken in results: 53 | results = results.split("\n") 54 | for i in range(len(results)): 55 | if "load average" in results[i]: 56 | break 57 | stats = dict() 58 | try: 59 | loadAverageLine = results[i] 60 | numbers = getAllNumbers(loadAverageLine) 61 | stats['1min_loadavg'] = numbers[-3] 62 | stats['5min_loadavg'] = numbers[-2] 63 | stats['15min_loadavg'] = numbers[-1] 64 | except Exception as e: 65 | print(e) 66 | try: 67 | stats['cpu_count'] = getFirstNumber(results[i+1]) 68 | stats['normalized_1min_loadavg'] = truncateFloat(stats['1min_loadavg'] / stats['cpu_count'], 3) 69 | stats['normalized_5min_loadavg'] = truncateFloat(stats['5min_loadavg'] / stats['cpu_count'], 3) 70 | stats['normalized_15min_loadavg'] = truncateFloat(stats['15min_loadavg'] / stats['cpu_count'], 3) 71 | except Exception as e: 72 | print(e) 73 | try: 74 | stats['mem_total'] = truncateFloat(getFirstNumber(results[i+2]) / 1e6, 2) 75 | except Exception as e: 76 | print(e) 77 | try: 78 | stats['mem_available'] = truncateFloat(getFirstNumber(results[i+3]) / 1e6, 2) 79 | except Exception as e: 80 | print(e) 81 | return stats 82 | else: 83 | return None 84 | 85 | 86 | def pipFreeze(*grep, logger=None, verbose=False): 87 | try: 88 | try: 89 | from pip._internal.operations import freeze 90 | except ImportError: # pip < 10.0 91 | from pip.operations import freeze 92 | x = list(freeze.freeze()) 93 | elements = set() 94 | for currentGrep in grep: 95 | print(currentGrep) 96 | if currentGrep is None: 97 | elements = elements.union(set(x)) 98 | else: 99 | for p in x: 100 | if currentGrep in p: 101 | elements.add(p) 102 | if verbose: 103 | if logger is not None: 104 | logger.log(str(elements)) 105 | else: 106 | print(str(elements)) 107 | return elements 108 | except Exception as e: 109 | if verbose: 110 | if logger is not None: 111 | logger.log(str(e)) 112 | else: 113 | print(str(e)) 114 | return None 115 | 116 | 117 | def isPlatform(text): 118 | if text is None: 119 | return False 120 | text = text.lower() 121 | currentPlatform = platform() 122 | currentPlatform = currentPlatform.lower() 123 | return currentPlatform.startswith(text) 124 | def windows(): 125 | return isPlatform("windows") 126 | def linux(): 127 | return isPlatform("linux") 128 | def mac(): 129 | return isPlatform("os") or isPlatform("mac") 130 | def platform(): 131 | platforms = { 132 | 'linux1' : 'Linux', 133 | 'linux2' : 'Linux', 134 | 'darwin' : 'OS X', 135 | 'win32' : 'Windows', 136 | 'nt' : 'Windows', 137 | 'win64' : 'Windows', 138 | } 139 | if sys.platform not in platforms: 140 | return sys.platform 141 | return platforms[sys.platform] 142 | 143 | # def sh(*args, **kwargs): 144 | # return exec(*args, **kwargs) 145 | def bash(*args, **kwargs): 146 | return exec(*args, **kwargs) 147 | def exec(commands, doPrint=True, useBuiltin=False, logger=None, verbose=True): 148 | """ 149 | Execute any command as a bash script for Linux platforms. 150 | In case useBuiltin is `True`, no output will be returned. 151 | For Linux platforms, the function will source `~/.bash_profile`, `~/.bashrc` and `~/.bash_aliases` if they exist. 152 | """ 153 | if linux(): 154 | if isinstance(commands, list): 155 | commands = "\n".join(commands) 156 | script = "" 157 | if isFile(homeDir() + "/.bashrc"): 158 | script += "source ~/.bashrc" + "\n" 159 | if isFile(homeDir() + "/.hjbashrc"): 160 | script += "source ~/.hjbashrc" + "\n" 161 | if isFile(homeDir() + "/.bash_profile"): 162 | script += "source ~/.bash_profile" + "\n" 163 | if isFile(homeDir() + "/.bash_aliases"): 164 | script += "shopt -s expand_aliases" + "\n" 165 | script += "source ~/.bash_aliases" + "\n" 166 | script += commands 167 | scriptPath = strToTmpFile(script) 168 | result = None 169 | try: 170 | # result = subprocess.Popen(["bash", scriptPath], shell=False, 171 | # stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) 172 | if doPrint and useBuiltin: 173 | os.system("bash " + scriptPath) 174 | else: 175 | # result = os.popen("bash " + scriptPath).read() 176 | sp = subprocess.Popen(["bash", scriptPath], shell=False, 177 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 178 | result, error = sp.communicate() 179 | result = byteToStr(result) 180 | error = byteToStr(error) 181 | if error is not None: 182 | result += "\n\n" + error 183 | result = result.strip() 184 | # result = sh.bash(scriptPath) 185 | except Exception as e: 186 | result = "Exception type: " + str(type(e)) + "\n" 187 | result += "Exception: " + str(e) 188 | try: 189 | result += "\n" + traceback.format_exc() 190 | except: pass 191 | rm(scriptPath) 192 | if doPrint and verbose and result is not None and len(result) > 0: 193 | if logger is None: 194 | print(result) 195 | else: 196 | logger.log(result) 197 | return result 198 | elif windows(): 199 | raise Exception("Please implement exec funct for Windows platform") 200 | elif mac(): 201 | raise Exception("Please implement exec funct for OS X platform") 202 | else: 203 | raise Exception("Unkown platform " + str(platform())) 204 | 205 | 206 | def getUsedPorts(logger=None): 207 | try: 208 | ports = [] 209 | result = sh.netstat("-plntu") 210 | if result is not None and len(result) > 0: 211 | for line in result.splitlines(): 212 | try: 213 | line = line.split() 214 | line = line[3].split(":") 215 | port = line[-1] 216 | port = int(port) 217 | ports.append(port) 218 | except Exception as e: 219 | if logger is not None: 220 | logger.error(str(e)) 221 | return ports 222 | except Exception as e: 223 | if logger is not None: 224 | logger.error(str(e)) 225 | return [] 226 | 227 | def isUser(text): 228 | return getUser().startswith(text) 229 | 230 | def getUser(): 231 | return getpass.getuser() 232 | 233 | def callTimeoutHandler(signum, frame): 234 | raise Exception("Function call timeout.") 235 | def setFunctionTimeout(*args, **kwargs): 236 | setCallTimeout(*args, **kwargs) 237 | def setCallTimeout(timeout): 238 | signal.signal(signal.SIGALRM, callTimeoutHandler) 239 | signal.alarm(math.ceil(timeout)) 240 | def resetFunctionTimeout(*args, **kwargs): 241 | resetCallTimeout(*args, **kwargs) 242 | def resetCallTimeout(): 243 | setCallTimeout(0) 244 | 245 | def ramAmount(*args, **kwargs): 246 | return getRAMTotal(*args, **kwargs) 247 | def getRAMTotal(): 248 | """ 249 | Return a value in Go 250 | """ 251 | mem = virtual_memory() 252 | return int(mem.total / pow(1024, 3)) 253 | 254 | def getProcCount(*args, **kwargs): 255 | return cpuCount(*args, **kwargs) 256 | def cpuCount(): 257 | return multiprocessing.cpu_count() 258 | 259 | 260 | def disableWifi(): 261 | sh.nmcli("radio", "wifi", "off") 262 | print("WiFi disabled!") 263 | def enableWifi(): 264 | sh.nmcli("radio", "wifi", "on") 265 | time.sleep(5) 266 | print("WiFi enabled!") 267 | 268 | def isDocker(): 269 | return isDir("/hosthome") 270 | 271 | def isHostname(hostname): 272 | return getHostname().startswith(hostname) 273 | 274 | def getHostname(): 275 | return socket.gethostname() 276 | 277 | 278 | # Deprecated (use sh lib instead): 279 | def sleep(seconds): 280 | time.sleep(seconds) 281 | 282 | # # Deprecated (use sh lib instead): 283 | # def bash(text): 284 | # """ 285 | # Deprecated: use sh lib instead 286 | # But it doesn't work on eclipse, use instead a python command line 287 | # """ 288 | # text = text.split(" ") 289 | # return subprocess.call(text) 290 | 291 | # # Deprecated (use sh lib instead): 292 | # def bash2(text): 293 | # """ 294 | # Deprecated: use sh lib instead 295 | # But it doesn't work on eclipse, use instead a python command line 296 | # """ 297 | # os.system(text) 298 | 299 | # # Deprecated (use sh lib instead): 300 | # def bash3(text): 301 | # """ 302 | # Deprecated: use sh lib instead 303 | # But it doesn't work on eclipse, use instead a python command line 304 | # """ 305 | # # text = ['/bin/bash', '-c'] + text.split(" ") 306 | # text = text.split(" ") 307 | # return subprocess.check_output(text) 308 | 309 | # # Déprecated (use sh lib instead): 310 | # def bash4(text): 311 | # """ 312 | # Deprecated: use sh lib instead 313 | # But it doesn't work on eclipse, use instead a python command line 314 | # """ 315 | # # text = ['/bin/bash', '-c'] + text.split(" ") 316 | # text = text.split(" ") 317 | # pipe = subprocess.Popen(text, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, executable='/bin/bash') 318 | # stdout, stderr = pipe.communicate() 319 | # return stdout 320 | 321 | 322 | 323 | def enum(enumName, *listValueNames): 324 | """ 325 | http://sametmax.com/faire-des-enums-en-python/ 326 | >>> FeuCirculation.TOUT_ETEINT 327 | 4 328 | >>> FeuCirculation.dictReverse[FeuCirculation.TOUT_ETEINT] 329 | 'TOUT_ETEINT' 330 | """ 331 | # Une suite d'entiers, on en crée autant 332 | # qu'il y a de valeurs dans l'enum. 333 | listValueNumbers = list(range(len(listValueNames))) 334 | # création du dictionaire des attributs. 335 | # Remplissage initial avec les correspondances : valeur d'enum -> entier 336 | dictAttrib = dict( list(zip(listValueNames, listValueNumbers)) ) 337 | # création du dictionnaire inverse. entier -> valeur d'enum 338 | dictReverse = dict( list(zip(listValueNumbers, listValueNames)) ) 339 | # ajout du dictionnaire inverse dans les attributs 340 | dictAttrib["dictReverse"] = dictReverse 341 | # création et renvoyage du type 342 | mainType = type(enumName, (), dictAttrib) 343 | return mainType 344 | 345 | 346 | # psutil.virtual_memory()["available"] 347 | 348 | # def stout(): 349 | # """ 350 | # Return True if we are on stout 351 | # """ 352 | # return (psutil.cpu_count(logical=True) > 40) and ((psutil.virtual_memory().total / (1*(10**9))) > 40); 353 | 354 | def getMemoryUsage(d): 355 | return asizeof(d) 356 | 357 | def freeRAM(): 358 | """ 359 | return the actual free ram space in Go 360 | """ 361 | return truncateFloat(psutil.virtual_memory().available / (1*(10**9)), 2) 362 | 363 | def warnFreeRAM(logger=None, verbose=True): 364 | if verbose: 365 | fr = freeRAM() 366 | msg = str(fr) + "g of RAM remaining." 367 | if logger is None: 368 | print(msg) 369 | else: 370 | logger.log(msg) 371 | if fr < 2: 372 | msg = "WARNING: the remaining RAM is very low!" 373 | if logger is None: 374 | print(msg) 375 | else: 376 | logger.log(msg) 377 | 378 | 379 | def usedRAM(): 380 | return truncateFloat(psutil.virtual_memory().used / (1*(10**9)), 2) 381 | 382 | def getMemoryPercent(): 383 | return psutil.virtual_memory().percent 384 | 385 | def printMemoryPercent(): 386 | print("Memory usage: " + str(getMemoryPercent()) + "%") 387 | 388 | 389 | def randomSleep(min=0.1, max=None): 390 | if max is None: 391 | max = min + 0.2 * min 392 | sleepDuration = getRandomFloat(min, max, decimalMax=8) 393 | time.sleep(sleepDuration) 394 | return sleepDuration 395 | 396 | def isWorkingProxy(proxy, verbose=False): 397 | try: 398 | http_proxy = "http://" + proxy 399 | https_proxy = "https://" + proxy 400 | ftp_proxy = "ftp://" + proxy 401 | proxyDict = { 402 | "http" : http_proxy, 403 | "https" : https_proxy, 404 | "ftp" : ftp_proxy 405 | } 406 | for url in \ 407 | [ 408 | "https://www.wikipedia.org/", 409 | "https://www.python.org/", 410 | ]: 411 | r = requests.get\ 412 | ( 413 | url, 414 | proxies=proxyDict, 415 | timeout=10, 416 | ) 417 | if len(r.text) > 100: 418 | return True 419 | except Exception as e: 420 | if verbose: 421 | print(str(e) + "\n" + str(traceback.format_exc())) 422 | return False 423 | 424 | 425 | # Deprecated : use argparse instead 426 | def argvOptionsToDict(argv=None): 427 | """ 428 | This function convert a command in dict key values according to command options. 429 | If the function return None, it means the argv doesn't have a good format. 430 | 431 | :example: 432 | >>> argvOptionsToDict(argv=["thecommand", "-r", "r", "-a", "a"]) 433 | {'a': 'a', 'r': 'r', 'command': 'thecommand'} 434 | >>> argvOptionsToDict(argv=["thecommand", "r", "r"]) is None 435 | True 436 | >>> argvOptionsToDict(argv=["thecommand"]) 437 | {'command': 'thecommand'} 438 | >>> argvOptionsToDict(argv=["thecommand", "r"]) is None 439 | True 440 | >>> argvOptionsToDict(argv=["thecommand", "--abcd", "/abcd/e"]) 441 | {'abcd': '/abcd/e', 'command': 'thecommand'} 442 | """ 443 | if argv is None: 444 | argv = sys.argv 445 | argvDict = dict() 446 | if argv is None or len(argv) == 0 or len(argv) % 2 == 0: 447 | return None 448 | argvDict["command"] = argv[0] 449 | for i in range(1, len(argv), 2): 450 | current = argv[i] 451 | if len(current) == 2: 452 | if not current.startswith('-'): 453 | return None 454 | argvDict[str(current[1])] = argv[i + 1] 455 | elif len(current) >= 3: 456 | if not current.startswith('--'): 457 | return None 458 | argvDict[str(current[2:len(current)])] = argv[i + 1] 459 | else: 460 | return None 461 | return argvDict 462 | 463 | # def argvOptionsToDict(argv=None): 464 | # """ 465 | # This function convert a command in dict key values according to command options. 466 | # If the function return None, it means the argv doesn't have a good format. 467 | # 468 | # :example: 469 | # >>> argvOptionsToDict(argv=["thecommand", "-r", "r", "-a", "a"]) 470 | # {'a': 'a', 'r': 'r', '_command': 'thecommand'} 471 | # >>> argvOptionsToDict(argv=["thecommand"]) 472 | # {'_command': 'thecommand'} 473 | # >>> argvOptionsToDict(argv=["thecommand", "r"]) is None 474 | # {'abcd': '/abcd/e', '_command': 'thecommand'} 475 | # >>> argvOptionsToDict(argv=["thecommand", "--abcd", "/abcd/e"]) 476 | # {'abcd': '/abcd/e', '_command': 'thecommand'} 477 | # >>> argvOptionsToDict(argv=["thecommand", "abcd.md", "-r", "rrr", "afile.txt"]) 478 | # {'_others': ["abcd.md", "afile.txt"], '_command': 'thecommand'} 479 | # """ 480 | # if argv is None: 481 | # argv = sys.argv 482 | # argvDict = dict() 483 | # if argv is None or len(argv) == 0 or len(argv) % 2 == 0: 484 | # return None 485 | # argvDict["_command"] = argv[0] 486 | # for i in range(1, len(argv), 2): 487 | # current = argv[i] 488 | # if len(current) == 2: 489 | # if not current.startswith('-'): 490 | # return None 491 | # argvDict[str(current[1])] = argv[i + 1] 492 | # elif len(current) >= 3: 493 | # if not current.startswith('--'): 494 | # return None 495 | # argvDict[str(current[2:len(current)])] = argv[i + 1] 496 | # else: 497 | # return None 498 | # return argvDict 499 | 500 | # Get all index where there is "-". Jump after a "-": 501 | # optionIndexes = [] 502 | # jumpNext = False 503 | # currentIndex = 1 504 | # while currentIndex < len(argv): 505 | # currentArg = argv[currentIndex] 506 | # if jumpNext: 507 | # jumpNext = False 508 | # currentIndex += 1 509 | # else: 510 | # if currentArg.startswith("-") and currentIndex < len(argv) - 1: 511 | # optionIndexes.append(currentIndex) 512 | # currentIndex += 2 513 | # else: 514 | # currentIndex += 1 515 | # 516 | # print optionIndexes 517 | 518 | def html2png(urlOrPath, destPath=None, width=None, height=None): 519 | """ 520 | If this function doesn't word, try firefox command line. 521 | If there are an old firefox profile error, just do 522 | `rm -rf ~/.mozilla` if you are in a docker container...` 523 | """ 524 | assert urlOrPath is not None 525 | if not urlOrPath.startswith("htt") and not urlOrPath.startswith("file"): 526 | urlOrPath = "file://" + urlOrPath 527 | if urlOrPath.startswith("file"): 528 | assert isFile(urlOrPath.replace("file://", "")) 529 | size = "" 530 | if width is not None: 531 | size = str(width) 532 | if height is not None: 533 | size += "," + str(height) 534 | size = "--window-size=" + size 535 | start = "firefox -headless -screenshot" 536 | if destPath is None and urlOrPath.startswith("file"): 537 | destPath = urlOrPath.replace("file://", "") + ".png" 538 | assert destPath is not None 539 | command = start + " " + destPath + " " + urlOrPath + " " + size 540 | bash(command, verbose=False) 541 | return destPath 542 | 543 | 544 | def installSent2Vec(): 545 | try: 546 | import sent2vec 547 | except: 548 | # from systemtools.system import bash 549 | bash("pip install Cython") 550 | bash("git clone https://github.com/epfml/sent2vec.git") 551 | bash("pip install ./sent2vec/") 552 | remove("sent2vec") 553 | import sent2vec 554 | 555 | 556 | 557 | def cropPNG(path, dst=None): 558 | if dst is None: 559 | dst = path 560 | bash("convert " + path + " -trim " + dst + "", verbose=False) 561 | return dst 562 | 563 | 564 | def test1(): 565 | # This will give you [568, 905, 1114, 882, 1120, 899, 1074, 859, 1126, 900, 553] because 0 is a result of getRandomFloat() from 0.0 to 0.5, 10 is the result of 9.5 to 10.0 566 | count = [0] * 11 567 | for i in range(10000): 568 | current = int(round(getRandomFloat() * 10.0)) 569 | count[current] += 1 570 | print(count) 571 | # So the solution is current = randint(0, 10) or: 572 | count = [0] * 11 573 | for i in range(10000): 574 | current = int(round(getRandomFloat() * 11.0 - 0.5)) 575 | count[current] += 1 576 | print(count) 577 | 578 | 579 | def testHtmlToPNG(): 580 | html2png("/home/hayj/Workspace/Python/Utils/MachineLearning/machinelearning/attmap/template-test2.html") 581 | 582 | 583 | if __name__ == "__main__": 584 | testHtmlToPNG() 585 | # print(tipiNumber()) 586 | # print(getRAMTotal()) 587 | 588 | # TODO 589 | """def copyDirectory(src, dst, symlinks=False, ignore=None): 590 | newFolder = re.search("/(\w+)/?$", src).group(1) 591 | if dst[-1] != "/": 592 | dst += "/" 593 | newFolderPath = dst + newFolder + "/" 594 | shutil.rmtree(newFolderPath) 595 | mkdirIfNotExists(newFolderPath) 596 | for item in os.listdir(src): 597 | s = os.path.join(src, item) 598 | d = os.path.join(dst, item) 599 | if os.path.isdir(s): 600 | shutil.copytree(s, d, symlinks, ignore) 601 | else: 602 | shutil.copy2(s, d)""" 603 | 604 | 605 | -------------------------------------------------------------------------------- /systemtools/file.py: -------------------------------------------------------------------------------- 1 | # pew in st-venv python ~/Workspace/Python/Utils/SystemTools/systemtools/file.py 2 | 3 | import errno 4 | import os 5 | import pickle 6 | import re 7 | import shutil 8 | import string 9 | import subprocess 10 | import time 11 | import zipfile 12 | from enum import Enum 13 | 14 | from systemtools.basics import getRandomStr, printLTS 15 | from systemtools.location import (decomposePath, getDir, homeDir, isDir, 16 | isFile, owner, sortedGlob, tmpDir) 17 | from systemtools.number import * 18 | 19 | try: 20 | from distutils.dir_util import copy_tree 21 | except: 22 | pass 23 | import requests 24 | 25 | try: 26 | import xtract 27 | except: 28 | pass 29 | import bz2 30 | import codecs 31 | import getpass 32 | import gzip 33 | 34 | 35 | def getHumanSize(path): 36 | return getSize(path, unit="auto", humanReadable=True) 37 | 38 | 39 | def getSize(path, unit="b", humanReadable=False, decimal=2): 40 | def __convertSize(size, unit): 41 | unit = unit.lower() 42 | if unit in ["k", "ko", "kilo"]: 43 | size = size / 1024 44 | elif unit in ["m", "mo", "mega"]: 45 | size = size / 1024 / 1024 46 | elif unit in ["g", "go", "giga"]: 47 | size = size / 1024 / 1024 / 1024 48 | else: # unit in ['b', 'bytes'] 49 | pass 50 | return size 51 | 52 | size = None 53 | if isFile(path): 54 | size = os.path.getsize(path) 55 | size = __convertSize(size, unit) 56 | elif isDir(path): 57 | totalSize = 0 58 | for current in sortedGlob(path + "/*"): 59 | totalSize += getSize(current, unit="b") 60 | size = __convertSize(totalSize, unit) 61 | if unit in ["a", "auto", None]: 62 | tempSize = size 63 | for u in ["k", "m", "g"]: 64 | tempSize = tempSize / 1024 65 | if tempSize < 1024 and tempSize > 0: 66 | size = tempSize 67 | unit = u 68 | break 69 | if humanReadable: 70 | return str(truncateFloat(size, decimal)) + unit 71 | else: 72 | return size 73 | 74 | 75 | class TIMESPENT_UNIT(Enum): 76 | DAYS = 1 77 | HOURS = 2 78 | MINUTES = 3 79 | SECONDS = 4 80 | 81 | 82 | def getLastModifiedTimeSpent( 83 | path, timeSpentUnit=TIMESPENT_UNIT.HOURS, logger=None, verbose=True 84 | ): 85 | try: 86 | diff = time.time() - os.path.getmtime(path) 87 | if timeSpentUnit == TIMESPENT_UNIT.SECONDS: 88 | return diff 89 | diff = diff / 60.0 90 | if timeSpentUnit == TIMESPENT_UNIT.MINUTES: 91 | return diff 92 | diff = diff / 60.0 93 | if timeSpentUnit == TIMESPENT_UNIT.HOURS: 94 | return diff 95 | diff = diff / 24.0 96 | if timeSpentUnit == TIMESPENT_UNIT.DAYS: 97 | return diff 98 | except Exception as e: 99 | if logger is not None and verbose: 100 | logger.log(str(e)) 101 | return 0 102 | 103 | 104 | def purgeOldFiles(pattern, maxTimeSpent, timeSpentUnit=TIMESPENT_UNIT.SECONDS): 105 | allPlugins = sortedGlob(pattern) 106 | for current in allPlugins: 107 | timeSpent = getLastModifiedTimeSpent(current, timeSpentUnit) 108 | if timeSpent > maxTimeSpent: 109 | removeFile(current) 110 | 111 | 112 | def rename(src, dst): 113 | return os.rename(src, dst) 114 | 115 | 116 | def strToFileName(*args, **kwargs): 117 | return strToFilename(*args, **kwargs) 118 | 119 | 120 | def move(src, dst): 121 | return shutil.move(src, dst) 122 | 123 | 124 | def strToFilename(text): 125 | """ 126 | 127 | https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename 128 | """ 129 | text = text.replace(" ", "_") 130 | valid_chars = "-_.()%s%s" % (string.ascii_letters, string.digits) 131 | return "".join(c for c in text if c in valid_chars) 132 | 133 | 134 | def serialize(obj, path): 135 | if path.endswith(".gzip"): 136 | with gzip.open(path, "wb") as f: 137 | pickle.dump(obj, f) 138 | else: 139 | with open(path, "wb") as handle: 140 | pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL) 141 | 142 | 143 | def deserialize(path): 144 | if path.endswith(".gzip"): 145 | with gzip.open(path, "rb") as f: 146 | return pickle.load(f) 147 | else: 148 | with open(path, "rb") as handle: 149 | return pickle.load(handle) 150 | 151 | 152 | def serializeToStr(obj): 153 | # https://stackoverflow.com/questions/30469575/how-to-pickle-and-unpickle-to-portable-string-in-python-3 154 | return codecs.encode(pickle.dumps(obj), "base64").decode() 155 | 156 | 157 | def deserializeFromStr(obj): 158 | return pickle.loads(codecs.decode(obj.encode(), "base64")) 159 | 160 | 161 | def copyDir(src, dst): 162 | if not (src.startswith("/") and src.startswith("/")): 163 | raise Exception("Pls give absolute path.") 164 | if src.count("/") < 2 or dst.count("/") < 2: 165 | raise Exception("Pls give a deep folder (by security).") 166 | (dir, filename, ext, filenameExt) = decomposePath(src) 167 | if dir[-1] != "/": 168 | dir += "/" 169 | dir = dir + filenameExt 170 | if dir != src or not isDir(dir): 171 | raise Exception("Pls give a right dir path.") 172 | dirName = dir.split("/")[-1] 173 | return copy_tree(src, dst + "/" + dirName) 174 | 175 | 176 | def copyFile(src, dst): 177 | """ 178 | Copy the file src to the file or directory dst. If dst is a directory, a file with the same basename as src is created (or overwritten) in the directory specified. Permission bits are copied. src and dst are path names given as strings. 179 | 180 | This function doesn't work when you give a file as the dst.... 181 | 182 | WARNING `copyfile` function doc: dst must be the complete target file name; look at shutil.copy() for a copy that accepts a target directory path 183 | """ 184 | return shutil.copy(src, dst) 185 | 186 | 187 | def getAllNumbers(text): 188 | """ 189 | This function is a copy of systemtools.basics.getAllNumbers 190 | """ 191 | if text is None: 192 | return None 193 | allNumbers = [] 194 | if len(text) > 0: 195 | # Remove space between digits : 196 | spaceNumberExists = True 197 | while spaceNumberExists: 198 | text = re.sub( 199 | "(([^.,0-9]|^)[0-9]+) ([0-9])", 200 | "\\1\\3", 201 | text, 202 | flags=re.UNICODE, 203 | ) 204 | if re.search("([^.,0-9]|^)[0-9]+ [0-9]", text) is None: 205 | spaceNumberExists = False 206 | numberRegex = "[-+]?[0-9]+[.,][0-9]+|[0-9]+" 207 | allMatchIter = re.finditer(numberRegex, text) 208 | if allMatchIter is not None: 209 | for current in allMatchIter: 210 | currentFloat = current.group() 211 | currentFloat = re.sub("\s", "", currentFloat) 212 | currentFloat = re.sub(",", ".", currentFloat) 213 | currentFloat = float(currentFloat) 214 | if currentFloat.is_integer(): 215 | allNumbers.append(int(currentFloat)) 216 | else: 217 | allNumbers.append(currentFloat) 218 | return allNumbers 219 | 220 | 221 | def mkdir(path): 222 | mkdirIfNotExists(path) 223 | 224 | 225 | def mkdirIfNotExists(path): 226 | """ 227 | This function make dirs recursively like mkdir -p in bash 228 | """ 229 | os.makedirs(path, exist_ok=True) 230 | 231 | 232 | def touch(fname, times=None): 233 | with open(fname, "a"): 234 | os.utime(fname, times) 235 | 236 | 237 | def replaceInFile(path, listSrc, listRep): 238 | with open(path, "r") as f: 239 | filedata = f.read() 240 | for i in range(len(listSrc)): 241 | src = listSrc[i] 242 | rep = listRep[i] 243 | filedata = filedata.replace(src, rep) 244 | with open(path, "w") as f: 245 | f.write(filedata) 246 | 247 | 248 | def fileExists(filePath): 249 | return os.path.exists(filePath) 250 | 251 | 252 | def globRemove(globPattern): 253 | filesPaths = sortedGlob(globPattern) 254 | removeFiles(filesPaths) 255 | 256 | 257 | def removeFile(path): 258 | print("DEPRECATED file or dir removal") 259 | if not isinstance(path, list): 260 | path = [path] 261 | for currentPath in path: 262 | try: 263 | os.remove(currentPath) 264 | except OSError: 265 | pass 266 | 267 | 268 | def removeFiles(path): 269 | print("DEPRECATED file or dir removal") 270 | removeFile(path) 271 | 272 | 273 | def removeAll(path): 274 | print("DEPRECATED file or dir removal") 275 | removeFile(path) 276 | 277 | 278 | def fileToStr(path, split=False, encoding=None): 279 | if split: 280 | return fileToStrList(path) 281 | else: 282 | with open(path, "r", encoding=encoding) as myfile: 283 | data = myfile.read() 284 | return data 285 | 286 | 287 | def fileToStrList(*args, removeDuplicates=False, **kwargs): 288 | result = fileToStrListYielder(*args, **kwargs) 289 | if removeDuplicates: 290 | return list(set(list(result))) 291 | else: 292 | return list(result) 293 | 294 | 295 | def basicLog(text, logger, verbose): 296 | if verbose: 297 | if text is not None and text != "": 298 | if logger is None: 299 | print(text) 300 | else: 301 | logger.info(text) 302 | 303 | 304 | def fileToStrListYielder( 305 | path, 306 | strip=True, 307 | skipBlank=True, 308 | commentStart="###", 309 | logger=None, 310 | verbose=False, 311 | ): 312 | 313 | if path is not None and isFile(path): 314 | commentCount = 0 315 | with open(path) as f: 316 | for line in f.readlines(): 317 | isComment = False 318 | if strip: 319 | line = line.strip() 320 | if ( 321 | commentStart is not None 322 | and len(commentStart) > 0 323 | and line.startswith(commentStart) 324 | ): 325 | commentCount += 1 326 | isComment = True 327 | if not isComment: 328 | if skipBlank and len(line) == 0: 329 | pass 330 | else: 331 | yield line 332 | if commentCount > 0: 333 | basicLog( 334 | "We found " + str(commentCount) + " comments in " + path, 335 | logger, 336 | verbose, 337 | ) 338 | else: 339 | basicLog(str(path) + " file not found.", logger, verbose) 340 | 341 | 342 | def linesCount(filePath): 343 | count = 0 344 | with open(filePath, "r") as f: 345 | for line in f: 346 | count += 1 347 | return count 348 | 349 | 350 | def rm(*args, **kwargs): 351 | return remove(*args, **kwargs) 352 | 353 | 354 | def remove( 355 | path, 356 | secure=True, 357 | minSlashCount=5, 358 | doRaise=True, 359 | skipDirs=False, 360 | skipFiles=False, 361 | decreaseMinSlashCountForTmp=True, 362 | ): 363 | if path is None or len(path) == 0: 364 | return 365 | if isinstance(path, str): 366 | path = [path] 367 | for currentPath in path: 368 | if secure and decreaseMinSlashCountForTmp and minSlashCount > 0: 369 | minSlashCount -= minSlashCount 370 | if secure and currentPath.count("/") < minSlashCount: 371 | errorMsg = "Not enough slashes in " + currentPath 372 | if doRaise: 373 | raise Exception(errorMsg) 374 | else: 375 | print(errorMsg) 376 | return 377 | if isDir(currentPath) and not skipDirs: 378 | try: 379 | return shutil.rmtree(currentPath, True) 380 | except Exception as e: 381 | if doRaise: 382 | raise e 383 | else: 384 | print(str(e)) 385 | if isFile(currentPath) and not skipFiles: 386 | try: 387 | os.remove(currentPath) 388 | except OSError as e: # this would be "except OSError, e:" before Python 2.6 389 | if ( 390 | e.errno != errno.ENOENT 391 | ): # errno.ENOENT = no such file or directory 392 | raise # re-raise exception if a different error occurred 393 | return 394 | 395 | 396 | def removeIfExists(path): 397 | # print("DEPRECATED file or dir removal") 398 | # try: 399 | # os.remove(path) 400 | # except OSError as e: # this would be "except OSError, e:" before Python 2.6 401 | # if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory 402 | # raise # re-raise exception if a different error occurred 403 | remove(path) 404 | 405 | 406 | def removeIfExistsSecure(path, slashCount=5): 407 | # print("DEPRECATED file or dir removal") 408 | # if path.count('/') >= slashCount: 409 | # removeIfExists(path) 410 | remove(path, minSlashCount=slashCount) 411 | 412 | 413 | def removeDir(*args, **kwargs): 414 | print("DEPRECATED file or dir removal") 415 | return removeTreeIfExists(*args, **kwargs) 416 | 417 | 418 | def removeTreeIfExists(path): 419 | print("DEPRECATED file or dir removal") 420 | return shutil.rmtree(path, True) 421 | 422 | 423 | def removeDirSecure(*args, **kwargs): 424 | print("DEPRECATED file or dir removal") 425 | return removeTreeIfExistsSecure(*args, **kwargs) 426 | 427 | 428 | def removeTreeIfExistsSecure(path, slashCount=5): 429 | # print("DEPRECATED file or dir removal") 430 | # if path.count('/') >= slashCount: 431 | # return removeTreeIfExists(path) 432 | # return None 433 | remove(path, minSlashCount=slashCount) 434 | 435 | 436 | def strListToTmpFile(theList, *args, **kwargs): 437 | text = "" 438 | for current in theList: 439 | text += current + "\n" 440 | return strToTmpFile(text, *args, **kwargs) 441 | 442 | 443 | def strToTmpFile(text, name=None, ext="", addRandomStr=False, *args, **kwargs): 444 | if text is None: 445 | text = "" 446 | if ext is None: 447 | ext = "" 448 | if ext != "": 449 | if not ext.startswith("."): 450 | ext = "." + ext 451 | if name is None: 452 | name = getRandomStr() 453 | elif addRandomStr: 454 | name += "-" + getRandomStr() 455 | path = tmpDir(*args, **kwargs) + "/" + name + ext 456 | strToFile(text, path) 457 | return path 458 | 459 | 460 | def strToFileAppend(*args, **kwargs): 461 | appendFile(*args, **kwargs) 462 | 463 | 464 | def appendToFile(*args, **kwargs): 465 | appendFile(*args, **kwargs) 466 | 467 | 468 | def appendStrToFile(*args, **kwargs): 469 | appendFile(*args, **kwargs) 470 | 471 | 472 | def appendFile(text, path, addBreakLine=True): 473 | if text is None: 474 | return 475 | if isinstance(text, list): 476 | text = "\n".join(text) 477 | with open(path, "a") as f: 478 | if addBreakLine: 479 | text = "\n" + str(text) 480 | f.write(text) 481 | 482 | 483 | def strListToFile(*args, **kwargs): 484 | strToFile(*args, **kwargs) 485 | 486 | 487 | def strToFile(text, path): 488 | # if not isDir(getDir(path)) and isDir(getDir(text)): 489 | # path, text = text, path 490 | if isinstance(text, list): 491 | text = "\n".join(text) 492 | with open(path, "w") as f: 493 | f.write(text) 494 | 495 | 496 | def normalizeNumericalFilePaths(globRegex): 497 | """ 498 | This function get a glob path and rename all file1.json file2.json ... file20.json 499 | to file01.json file02.json ... file20.json to better sort the folder by file names 500 | """ 501 | # We get all paths: 502 | allPaths = sortedGlob(globRegex) 503 | allNumbers = [] 504 | # We get all ints: 505 | for path in allPaths: 506 | # Get the filename without extension: 507 | (dir, filename, ext, filenameExt) = decomposePath(path) 508 | # Get all numbers: 509 | currentNumbers = getAllNumbers(filename) 510 | # Check if we have a int first: 511 | if currentNumbers is None or len(currentNumbers) == 0: 512 | print("A filename has no number.") 513 | return False 514 | firstNumber = currentNumbers[0] 515 | if not isinstance(firstNumber, int): 516 | print("A filename has no float as first number.") 517 | return False 518 | # Add it in the list: 519 | allNumbers.append(firstNumber) 520 | # Get the max int: 521 | maxInt = max(allNumbers) 522 | # Calculate the nmber of digit: 523 | digitCountHasToBe = len(str(maxInt)) 524 | # Replace all : 525 | i = 0 526 | for i in range(len(allNumbers)): 527 | currentPath = allPaths[i] 528 | (dir, filename, ext, filenameExt) = decomposePath(currentPath) 529 | currentInt = allNumbers[i] 530 | currentRegex = "0*" + str(currentInt) 531 | zerosCountToAdd = digitCountHasToBe - len(str(currentInt)) 532 | zerosStr = "0" * zerosCountToAdd 533 | newFilename = re.sub( 534 | currentRegex, zerosStr + str(currentInt), filename, count=1 535 | ) 536 | newFilename = dir + newFilename + "." + ext 537 | if currentPath != newFilename: 538 | os.rename(currentPath, newFilename) 539 | print(newFilename + " done.") 540 | i += 1 541 | return True 542 | 543 | 544 | def encryptFile( 545 | path, 546 | key, 547 | text=None, 548 | ext=".encrypted.zip", 549 | remove=False, 550 | logger=None, 551 | verbose=True, 552 | ): 553 | """ 554 | This function encrypt a file, if you give text in `text` parameter, 555 | the function will create the file. 556 | Return True if all is ok. 557 | """ 558 | try: 559 | if text is not None: 560 | strToFile(text, path) 561 | rc = subprocess.call( 562 | ["7z", "a", "-p" + key, "-y", path + ext, path], 563 | stdout=open(os.devnull, "wb"), 564 | ) 565 | if remove: 566 | removeFile(path) 567 | return True 568 | except Exception as e: 569 | if verbose: 570 | if logger is None: 571 | print(str(e)) 572 | else: 573 | logger.error(str(e)) 574 | return False 575 | 576 | 577 | def fileToMultiParts( 578 | filePath, outputDir=None, nbParts=40, compress=True, checkLineCount=False 579 | ): 580 | """ 581 | This function take a file path and write it in muliparts 582 | """ 583 | if outputDir is None: 584 | outputDir = filePath + "-multiparts" 585 | if not isDir(outputDir): 586 | mkdir(outputDir) 587 | if checkLineCount: 588 | print(filePath) 589 | c = linesCount(filePath) 590 | if nbParts > c: 591 | nbParts = c 592 | with open(filePath, "r") as f: 593 | if compress: 594 | files = [ 595 | bz2.open(outputDir + "/%d.txt.bz2" % i, "wt") 596 | for i in range(nbParts) 597 | ] 598 | else: 599 | files = [ 600 | open(outputDir + "/%d.txt" % i, "w") for i in range(nbParts) 601 | ] 602 | for i, line in enumerate(f): 603 | files[i % nbParts].write(line) 604 | for f in files: 605 | f.close() 606 | return outputDir 607 | 608 | 609 | def decryptFile( 610 | path, key, ext=".encrypted.zip", remove=False, logger=None, verbose=True 611 | ): 612 | """ 613 | This function decrypt a file and return the text 614 | """ 615 | try: 616 | (dir, _, _, _) = decomposePath(path) 617 | key = str.encode(key) 618 | if path[-len(ext) :] != ext: 619 | decryptedFilePath = path 620 | cryptedFilePath = decryptedFilePath + ext 621 | else: 622 | cryptedFilePath = path 623 | decryptedFilePath = path[: -len(ext)] 624 | zipfile.ZipFile(cryptedFilePath).extractall(dir, None, key) 625 | if remove: 626 | removeFile(cryptedFilePath) 627 | return fileToStr(decryptedFilePath) 628 | except Exception as e: 629 | if verbose: 630 | if logger is None: 631 | print(str(e)) 632 | else: 633 | logger.error(str(e)) 634 | return None 635 | 636 | 637 | def download(url, dirPath=None, skipIfExists=False): 638 | """ 639 | Based on https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py/39217788 640 | """ 641 | if dirPath is None: 642 | dirPath = tmpDir("downloads") 643 | fileName = strToFilename(url.split("/")[-1]) 644 | filePath = dirPath + "/" + fileName 645 | if skipIfExists and isFile(filePath): 646 | return filePath 647 | else: 648 | r = requests.get(url, stream=True) 649 | with open(filePath, "wb") as f: 650 | for chunk in r.iter_content(chunk_size=1024): 651 | if chunk: 652 | f.write(chunk) 653 | return filePath 654 | 655 | 656 | def extract( 657 | filePath, destinationDir=None, upIfUnique=True, doDoubleExtract=True 658 | ): 659 | if not isFile(filePath): 660 | print(filePath + " does not exist") 661 | return None 662 | # We get the dir of the file to extract: 663 | (dirPath, _, _, filenameExt) = decomposePath(filePath) 664 | # We extract it: 665 | extractedDirPath = xtract.xtract(filePath) 666 | # Here we check if the file end with ".tar": 667 | if doDoubleExtract and extractedDirPath[-4:] == ".tar": 668 | # So we re-extract it: 669 | previousPath = extractedDirPath 670 | extractedDirPath = xtract.xtract(extractedDirPath) 671 | # We remove the previous element: 672 | if isDir(previousPath): 673 | remove(previousPath, minSlashCount=4) 674 | elif isFile(previousPath): 675 | remove(previousPath, minSlashCount=4) 676 | # If there is only one folder or file under extractedDirPath, we up it: 677 | if upIfUnique and len(sortedGlob(extractedDirPath + "/*")) == 1: 678 | # We get the element path: 679 | elementPath = sortedGlob(extractedDirPath + "/*")[0] 680 | # We make the dst path: 681 | dst = dirPath + "/" + elementPath.split("/")[-1] 682 | # First we check if the element exists inthe parent dir: 683 | if isFile(dst) or isDir(dst): 684 | dst += time.strftime("-%Y.%m.%d-%H.%M.%S") 685 | # then we move it: 686 | shutil.move(elementPath, dst) 687 | # And finally we remove the dir: 688 | remove(extractedDirPath, minSlashCount=4) 689 | # We update extractedDirPath: 690 | extractedDirPath = dst 691 | # We move the element: 692 | if destinationDir is not None: 693 | # We move it: 694 | newDestFilePath = ( 695 | destinationDir + "/" + decomposePath(extractedDirPath)[3] 696 | ) 697 | shutil.move(extractedDirPath, newDestFilePath) 698 | # We update extractedDirPath: 699 | extractedDirPath = newDestFilePath 700 | # Finally we return the new path: 701 | return extractedDirPath 702 | 703 | 704 | def testFileToMultiParts(): 705 | directory = getExecDir(__file__) + "/testdata" 706 | filePath = sortedGlob(directory + "/*")[0] 707 | workingDir = tmpDir("vectors-test") 708 | result = extract(filePath, destinationDir=workingDir) 709 | outputDir = fileToMultiParts(result, checkLineCount=True, compress=True) 710 | print(outputDir) 711 | 712 | 713 | def testSizeHumanSize(): 714 | path = "/home/hayj/tmp/WordVectors/fasttext/crawl-300d-2M.vec" 715 | path = "/home/hayj/tmp/d2v" 716 | path = "/home/hayj/tmp/psl.txt" 717 | print(os.path.getsize(path)) 718 | print(getSize(path, humanReadable=True, unit="m")) 719 | print(getHumainSize(path)) 720 | 721 | 722 | def clearRtmp(*args, **kwargs): 723 | return cleanRtmp(*args, **kwargs) 724 | 725 | 726 | def cleanRtmp(*args, **kwargs): 727 | if len(args) == 0: 728 | args = ("/tmp",) 729 | return cleanDir(*args, **kwargs) 730 | 731 | 732 | def cleanDir( 733 | path, 734 | startsWith=None, 735 | endsWith=None, 736 | olderHour=4, 737 | onlyOwner=True, 738 | verbose=False, 739 | logger=None, 740 | dryRun=False, 741 | removeKwargs={}, 742 | pathContains="/tmp", # For security purpose 743 | ): 744 | me = getpass.getuser() 745 | elementsToDelete = [] 746 | for element in sortedGlob(path + "/*"): 747 | if onlyOwner and owner(element) != me: 748 | continue 749 | if ( 750 | olderHour is not None 751 | and getLastModifiedTimeSpent( 752 | element, 753 | timeSpentUnit=TIMESPENT_UNIT.HOURS, 754 | logger=logger, 755 | verbose=False, 756 | ) 757 | < olderHour 758 | ): 759 | continue 760 | if startsWith is not None and not decomposePath(element)[3].startswith( 761 | startsWith 762 | ): 763 | continue 764 | if endsWith is not None and not decomposePath(element)[3].endswith( 765 | endsWith 766 | ): 767 | continue 768 | elementsToDelete.append(element) 769 | for element in elementsToDelete: 770 | if pathContains in element: 771 | try: 772 | if not dryRun: 773 | if "secure" not in removeKwargs: 774 | removeKwargs["secure"] = False 775 | remove(element, **removeKwargs) 776 | if verbose: 777 | msg = "We removed " + element 778 | if logger is not None: 779 | try: 780 | logger.log(msg) 781 | except: 782 | pass 783 | else: 784 | print(msg) 785 | except Exception as e: 786 | print(e) 787 | 788 | 789 | if __name__ == "__main__": 790 | # testRM() 791 | # print(download("http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz")) 792 | # extract("/home/hayj/tmp/downloads/aclImdb_v1.tar.gz") 793 | # print(extract("/home/hayj/tmp/downloads/aclImdb_v1.tar.gz", tmpDir("aaa"))) 794 | # normalizeNumericalFilePaths("/home/hayj/test/test1/*.txt") 795 | # normalizeNumericalFilePaths("/users/modhel/hayj/NoSave/Data/TwitterArchiveOrg/Converted/*.bz2") 796 | # strToTmpFile("hoho", subDir="test", ext="txt") 797 | # strToFile("haha", tmpDir(subDir="test") + "/test.txt") 798 | 799 | # key = 'AAA' 800 | # text = "bbb" 801 | # print(encryptFile(homeDir() + '/tmp/titi.txt', key, text=text)) 802 | # 803 | # 804 | # text = decryptFile(homeDir() + '/tmp/titi.txt', key) 805 | # 806 | # print(text) 807 | cleanDir(tmpDir(), startsWith=None, olderHour=4, verbose=True, dryRun=True) 808 | --------------------------------------------------------------------------------