├── test ├── __init__.py ├── test_fast │ ├── __init__.py │ └── test_language_guessing.py └── test_slow │ ├── __init__.py │ └── test_classify_all.py ├── whats_that_code ├── __init__.py ├── commands │ └── __init__.py ├── _version.py ├── guess_by_popularity.py ├── guess_by_code_and_tags.py ├── __main__.py ├── regex_based.py ├── extension_based.py ├── tag_based.py ├── pygments_based.py ├── shebang_based.py ├── parsing_based.py ├── election.py ├── codex_markers.py └── known_languages.py ├── navio_tasks ├── commands │ ├── __init__.py │ ├── README.md │ ├── cli_pyt.py │ ├── cli_npm_pyright.py │ ├── cli_bandit.py │ ├── cli_django_tests.py │ ├── cli_go_gitleaks.py │ ├── cli_compile_py.py │ ├── cli_tox.py │ ├── cli_detect_secrets.py │ ├── cli_get_secrets.py │ ├── cli_mypy.py │ ├── cli_pytest.py │ └── cli_pylint.py ├── dependency_commands │ ├── __init__.py │ ├── cli_safety.py │ ├── cli_pip.py │ ├── cli_pin_dependencies.py │ └── cli_installs.py ├── mutating_commands │ ├── __init__.py │ ├── cli_npm_prettier.py │ ├── README.md │ ├── cli_isort.py │ ├── cli_pyupgrade.py │ ├── cli_jiggle_version.py │ ├── cli_black.py │ └── cli_precommit.py ├── non_breaking_commands │ ├── __init__.py │ ├── README.md │ ├── cli_mccabe.py │ ├── cli_vulture.py │ ├── cli_scspell.py │ └── cli_sonar.py ├── packaging_commands │ ├── cli_poetry_package.py │ ├── __init__.py │ ├── README.md │ └── cli_twine.py ├── __init__.py ├── pure_reports │ ├── __init__.py │ ├── README.md │ ├── cli_sphinx.py │ ├── cli_gitchangelog.py │ └── cli_pygount.py ├── non_py_commands │ ├── __init__.py │ ├── cli_yamllint.py │ └── cli_openapi.py ├── file_system.py ├── clean.py ├── network.py ├── output.py ├── README.md ├── utils.py ├── system_info.py ├── settings.py ├── cli_commands.py └── build_state.py ├── .config ├── vendorize.toml ├── run_with_docker.sh ├── docker.sh ├── dockerw.sh ├── spelling.dic ├── .flake8 ├── Dockerfile ├── requirements.txt ├── .pynt ├── requirements-dev.txt ├── requirements_for_safety.txt ├── .license_rules └── .pylintrc ├── docs ├── CHANGES.md ├── CONTRIBUTING.md ├── TODO.md └── prior_art.md ├── .env_example ├── .dockerignore ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── tox.ini ├── .editorconfig ├── Pipfile ├── .pre-commit-config.yaml ├── LICENSE ├── scripts └── get_most_popular.py ├── .github └── workflows │ ├── build.yml │ └── codeql.yml ├── .gitignore ├── README.md └── pyproject.toml /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_fast/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_slow/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /whats_that_code/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /navio_tasks/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /whats_that_code/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /navio_tasks/dependency_commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /navio_tasks/packaging_commands/cli_poetry_package.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run `poetry build` 3 | """ 4 | -------------------------------------------------------------------------------- /.config/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "hello/_vendor" 2 | packages = [ 3 | "jiggle_version", 4 | ] 5 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_npm_prettier.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reformat with prettier, which is a node tool. 3 | """ 4 | -------------------------------------------------------------------------------- /docs/CHANGES.md: -------------------------------------------------------------------------------- 1 | Version 0.1.0, 2021-01-17 2 | ----- 3 | - Forked from so_pip 4 | - Bringing this into existance as its own thing. 5 | -------------------------------------------------------------------------------- /navio_tasks/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command line wrappers mostly 3 | """ 4 | 5 | from navio_tasks.settings import load_config 6 | 7 | load_config() 8 | -------------------------------------------------------------------------------- /whats_that_code/_version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Version file 3 | Starts at 0.1.0 4 | Increments to 0.1.1, etc on build 5 | """ 6 | 7 | __version__ = "0.1.4" 8 | -------------------------------------------------------------------------------- /.config/run_with_docker.sh: -------------------------------------------------------------------------------- 1 | ( 2 | export MSYS_NO_PATHCONV=1 3 | ./dockerw.sh run --rm --volume "$PWD":/output so_pip:latest "$@" --output=/output --logs 4 | ) 5 | -------------------------------------------------------------------------------- /navio_tasks/pure_reports/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Commands that related to packaging your code. 3 | If you are deploying with just copy, you don't need any packaging. 4 | """ 5 | -------------------------------------------------------------------------------- /navio_tasks/non_py_commands/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Commands that related to packaging your code. 3 | If you are deploying with just copy, you don't need any packaging. 4 | """ 5 | -------------------------------------------------------------------------------- /.config/docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # If you you gitbash, to use tflint, etc docker version, you need this. 4 | ( 5 | export MSYS_NO_PATHCONV=1 6 | "docker.exe" "$@" 7 | ) 8 | -------------------------------------------------------------------------------- /navio_tasks/packaging_commands/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Commands that related to packaging your code. 3 | If you are deploying with just copy, you don't need any packaging. 4 | """ 5 | -------------------------------------------------------------------------------- /.config/dockerw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # If you you gitbash, to use tflint, etc docker version, you need this. 4 | ( 5 | export MSYS_NO_PATHCONV=1 6 | "winpty" "docker.exe" "$@" 7 | ) 8 | -------------------------------------------------------------------------------- /.config/spelling.dic: -------------------------------------------------------------------------------- 1 | pylint 2 | PyLint 3 | flake8 4 | vendorizing 5 | pystackexchange 6 | pypi 7 | xlrpc 8 | api 9 | isort 10 | wictionary 11 | Matlab 12 | Batchfile 13 | Erlang 14 | erlang 15 | matlab 16 | -------------------------------------------------------------------------------- /.env_example: -------------------------------------------------------------------------------- 1 | export key= Register for a key here: https://stackapps.com/apps/oauth/register Do not call you "app" so_pip, use your user name. 2 | export PYTHONIOENCODING=utf-8 3 | export LC_ALL=en_US.UTF-8 4 | export LANG=en_US.UTF-8 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .build_state 2 | .mypy_cache 3 | .pytest_cache 4 | coverage 5 | # must include dist or can't install from dist folder! 6 | docs 7 | node_modules 8 | npm-debug.log 9 | Dockerfile* 10 | docker-compose* 11 | .dockerignore 12 | .git 13 | .gitignore 14 | README.md 15 | LICENSE 16 | .vscode 17 | -------------------------------------------------------------------------------- /.config/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E722,W503,E126,E122,E131,E203,E231,C901, 3 | DAR101,DAR201,DAR101,D202,D200,D400,D205,DAR301 4 | Q000,C812,C819, 5 | WPS306,WPS305,WPS336, WPS400,WPS110,WPS114, 6 | WPS214, 7 | E800,G004,G003,C416,E501,G200 8 | max-line-length = 88 9 | enable-extensions = G 10 | exclude = tests/* 11 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/README.md: -------------------------------------------------------------------------------- 1 | These commands change the source code. 2 | 3 | - On a build server, needing to change code should break the build. 4 | - After changing code, the new code needs to be checked in. 5 | - Mutating should happen before unit tests. 6 | - Black should be the final mutator because only one formatter can "win" 7 | -------------------------------------------------------------------------------- /navio_tasks/pure_reports/README.md: -------------------------------------------------------------------------------- 1 | These commands create some sort of report 2 | 3 | - Generally can't fail a build 4 | - Doesn't involve a TODO list of problems to work 5 | - Might be informational 6 | - Not a lot of value to run every build if no one looks at them. 7 | 8 | e.g. count line counts, generating change log, docs, authors file 9 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/README.md: -------------------------------------------------------------------------------- 1 | These commands create some problem list, but it doesn't make sense to fix the immediately 2 | 3 | Counter example- syntax errors must be fixed immediately or the app can't run at all. 4 | 5 | Spelling errors, dead code, duplicate code, "complex" code doesn't necessarily 6 | cause an immediately app failure, but might be worth working periodically. 7 | -------------------------------------------------------------------------------- /.config/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.0-slim-buster 2 | WORKDIR /app 3 | ENV PIP_NO_CACHE_DIR=off 4 | ENV PYTHONUNBUFFERED=1 5 | # RUN apk add --no-cache linux-headers==4.19.36-r0 wget 6 | RUN python -m pip install --no-cache-dir --upgrade pip --quiet \ 7 | && pip install pipenv --no-cache-dir --quiet 8 | COPY dist/* /app/ 9 | RUN pipenv install so_pip-*.whl --skip-lock 10 | ENTRYPOINT ["pipenv", "run", "so_pip"] 11 | -------------------------------------------------------------------------------- /navio_tasks/file_system.py: -------------------------------------------------------------------------------- 1 | """ 2 | File system related 3 | """ 4 | 5 | import os 6 | 7 | from navio_tasks.settings import PROBLEMS_FOLDER, REPORTS_FOLDER 8 | 9 | 10 | def initialize_folders() -> None: 11 | """ 12 | Create folders that are likely to be needed 13 | """ 14 | for folder in [PROBLEMS_FOLDER, REPORTS_FOLDER]: 15 | if not os.path.exists(folder): 16 | os.makedirs(folder) 17 | -------------------------------------------------------------------------------- /navio_tasks/packaging_commands/README.md: -------------------------------------------------------------------------------- 1 | If a packaging strategy is being used 2 | 3 | - Problems should fail a build if the app can't be packaged. 4 | 5 | Even if no packaging is being used 6 | 7 | - Dependencies need to be in place for a build to pass 8 | - Dependencies should be pinned tight enough to get repeatable builds 9 | - Not so tight that you can't get security fixes and fixes necessary to interact with 10 | a changing world. 11 | -------------------------------------------------------------------------------- /navio_tasks/commands/README.md: -------------------------------------------------------------------------------- 1 | All the commands in the folder 2 | 3 | - Break the build if there is a problem 4 | - Represent governance 5 | - Should each have a way to 6 | - fix a problem 7 | - ignore a problem, explicitly (ignore lint for 1 line of code) 8 | - allow for a certain number of problems (e.g. 10 lines of lint) 9 | 10 | 11 | It is fair to ask developers to make sure all of the tools in this folder 12 | say the code is secure and has no problems that can be detected. 13 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3/.devcontainer/base.Dockerfile 2 | 3 | # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster 4 | # FROM mcr.microsoft.com/devcontainers/python:3.11-bullseye 5 | FROM ghcr.io/matthewdeanmartin/convenient_py_devcontainer:latest 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # tox (https://tox.readthedocs.io/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | [tox] 6 | # vermin reports so_pip source itself is 3.7+ 7 | envlist = py36, py37, py38, py39, py310, py311, py312 8 | isolated_build = True 9 | [testenv] 10 | commands = 11 | pytest test/test_fast -v --quiet 12 | deps = 13 | pytest==7.2.1 14 | pytest-cov==4.0.0 15 | -r .config/requirements.txt 16 | -------------------------------------------------------------------------------- /navio_tasks/clean.py: -------------------------------------------------------------------------------- 1 | """ 2 | Removing old crap. 3 | 4 | Trivial except when something has a lock on a file. 5 | """ 6 | 7 | import os 8 | 9 | 10 | def clean_old_files() -> None: 11 | """ 12 | Output files all moved to /problems/ or /reports/, clean up 13 | detritus left by old versions of build.py 14 | """ 15 | for file in [ 16 | "dead_code.txt", 17 | "lint.txt", 18 | "mypy_errors.txt", 19 | "line_counts.txt", 20 | "manifest_errors.txt", 21 | "detect-secrets-results.txt", 22 | ]: 23 | if os.path.exists(file): 24 | os.remove(file) 25 | -------------------------------------------------------------------------------- /whats_that_code/guess_by_popularity.py: -------------------------------------------------------------------------------- 1 | """ 2 | Take everyone elses votes. If a vote is for something popular, vote for it in popularity 3 | rank 4 | """ 5 | 6 | from typing import List, Set 7 | 8 | from whats_that_code.known_languages import POPULARITY_LIST 9 | 10 | 11 | def language_by_popularity(guesses: Set[str]) -> List[str]: 12 | """Guess by parsing""" 13 | # TODO: maybe take into account ranking, use cutoffs. 14 | if not guesses: 15 | return [] 16 | votes = [] 17 | for guess in guesses: 18 | if guess in POPULARITY_LIST: 19 | votes.append(guess) 20 | return votes 21 | -------------------------------------------------------------------------------- /navio_tasks/non_py_commands/cli_yamllint.py: -------------------------------------------------------------------------------- 1 | """ 2 | Check yaml, but doesn't reformat it 3 | """ 4 | 5 | from navio_tasks.cli_commands import check_command_exists, execute 6 | from navio_tasks.settings import PROJECT_NAME, VENV_SHELL 7 | from navio_tasks.utils import inform 8 | 9 | 10 | def do_yamllint() -> str: 11 | """ 12 | Check yaml files for problems 13 | """ 14 | command = "yamllint" 15 | check_command_exists(command) 16 | 17 | command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace(" ", " ") 18 | inform(command) 19 | execute(*(command.split(" "))) 20 | return "yamllint succeeded" 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # See https://editorconfig.org for format details and 2 | # https://editorconfig.org/#download for editor / IDE integration 3 | 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | indent_size = 4 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | end_of_line = lf 12 | charset = utf-8 13 | 14 | [*.py] 15 | max_line_length = 88 16 | 17 | # Makefiles always use tabs for indentation 18 | [Makefile] 19 | indent_style = tab 20 | 21 | # We don't want to apply our defaults to third-party code or minified bundles: 22 | [**/{external,vendor}/**,**.min.{js,css}] 23 | indent_style = ignore 24 | indent_size = ignore 25 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_pyt.py: -------------------------------------------------------------------------------- 1 | """ 2 | Security tool. Unsupported since 3.9 3 | """ 4 | 5 | import sys 6 | 7 | from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple 8 | 9 | 10 | def do_python_taint() -> str: 11 | """ 12 | Security Checks 13 | """ 14 | if sys.version_info.major == 3 and sys.version_info.minor > 8: 15 | # pyt only works with python <3.8 16 | return "skipping python taint" 17 | 18 | command = "pyt" 19 | check_command_exists(command) 20 | command = "pyt -r" 21 | command = prepinform_simple(command) 22 | execute(*(command.split(" "))) 23 | return "ok" 24 | -------------------------------------------------------------------------------- /.config/requirements.txt: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # This requirements file has been automatically generated from `Pipfile` with 3 | # `pipenv-to-requirements` 4 | # 5 | # 6 | # This has been done to maintain backward compatibility with tools and services 7 | # that do not support `Pipfile` yet. 8 | # 9 | # Do NOT edit it directly, use `pipenv install [-d]` to modify `Pipfile` and 10 | # `Pipfile.lock` and then regenerate `requirements*.txt`. 11 | ################################################################################ 12 | 13 | defusedxml 14 | navio-builder-win>=0.1.57 15 | pygments 16 | pyrankvote 17 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | pyrankvote = "*" 8 | pygments = "*" 9 | defusedxml = "*" 10 | navio-builder-win = ">=0.1.57" 11 | 12 | [dev-packages] 13 | pytest = "*" 14 | python-dotenv = "*" 15 | pyrankvote = "*" 16 | vermin = "*" 17 | ifaddr = "*" 18 | psutil = "*" 19 | pebble = "*" 20 | sshtunnel = "*" 21 | tox = "*" 22 | checksumdir = "*" 23 | pytest-cov = "*" 24 | pytest-timeout = "*" 25 | pytest-xdist = "*" 26 | gitpython = "*" 27 | wheel = "*" 28 | requests = "*" 29 | beautifulsoup4 = "*" 30 | mypy = "*" 31 | pre-commit = "*" 32 | 33 | [requires] 34 | python_version = "3.11" 35 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | Clone, pipenv, run `python -m navio package` 3 | 4 | See [TODO.md](TODO.md) for roadmap. 5 | 6 | ## Training Sets 7 | - [Rosetta Code](http://www.rosettacode.org/wiki/Rosetta_Code) 8 | - [GuessLangTools](https://github.com/yoeo/guesslangtools) Tool for downloading examples in bulk 9 | - Also see [Prior Art page](prior_art.md), many of those tools posted their training data. 10 | 11 | ## Input data 12 | - [Reserved Keywords](https://github.com/matthewdeanmartin/Reserved-Key-Words-list-of-various-programming-languages) 13 | - [Languages + Extensions](https://gist.github.com/aymen-mouelhi/82c93fbcd25f091f2c13faa5e0d61760) in JSON form 14 | -------------------------------------------------------------------------------- /navio_tasks/pure_reports/cli_sphinx.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generate docs based on rst. 3 | """ 4 | 5 | from navio_tasks.cli_commands import ( 6 | check_command_exists, 7 | config_pythonpath, 8 | execute_with_environment, 9 | ) 10 | from navio_tasks.settings import VENV_SHELL 11 | from navio_tasks.utils import inform 12 | 13 | 14 | def do_docs() -> str: 15 | """ 16 | Generate docs based on rst. 17 | """ 18 | check_command_exists("make") 19 | 20 | my_env = config_pythonpath() 21 | command = f"{VENV_SHELL} make html".strip().replace(" ", " ") 22 | inform(command) 23 | execute_with_environment(command, env=my_env) 24 | return "Docs generated" 25 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v6.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | exclude: ^docs/conf.py$ 10 | - repo: https://github.com/psf/black-pre-commit-mirror 11 | rev: 25.12.0 12 | hooks: 13 | - id: black 14 | args: [--target-version, py38] 15 | - repo: https://github.com/asottile/pyupgrade 16 | rev: v3.21.2 17 | hooks: 18 | - id: pyupgrade 19 | args: [--py38-plus] 20 | #- repo: https://github.com/PyCQA/isort 21 | # rev: 5.6.4 22 | # hooks: 23 | # - id: isort 24 | -------------------------------------------------------------------------------- /.config/.pynt: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | PROJECT_NAME = whats_that_code 3 | SRC = . 4 | PROBLEMS_FOLDER = reports 5 | REPORTS_FOLDER = reports 6 | VENV_SHELL = poetry run 7 | 8 | MINIMUM_TEST_COVERAGE = 20 9 | 10 | # if 1, disables bandit. Bandit disallows all shell commands. 11 | IS_SHELL_SCRIPT_LIKE = 0 12 | 13 | ; Below this there should be 0 lint, 0 mypy, etc. 14 | ; Above this follow maximum lint 15 | SMALL_CODE_BASE_CUTOFF = 50 16 | 17 | MAXIMUM_LINT = 6 18 | MAXIMUM_MYPY = 16 19 | MAXIMUM_DEAD_CODE = 500 20 | MAXIMUM_MANIFEST_ERRORS = 0 21 | 22 | COMPLEXITY_CUT_OFF = 16 23 | 24 | DOCKER_TAG=so_pip:pynt 25 | 26 | KNOWN_IP_PREFIX=111.111 27 | SPEAK_WHEN_BUILD_FAILS=True 28 | RUN_ALL_TESTS_REGARDLESS_TO_NETWORK=True 29 | PACKAGE_WITH=poetry 30 | INSTALL_WITH=pipenv 31 | -------------------------------------------------------------------------------- /whats_that_code/guess_by_code_and_tags.py: -------------------------------------------------------------------------------- 1 | """ 2 | What programming language is this text 3 | """ 4 | 5 | from typing import List, Tuple 6 | 7 | from whats_that_code.election import guess_language_all_methods 8 | from whats_that_code.known_languages import FILE_EXTENSIONS 9 | 10 | GUESS = None 11 | 12 | 13 | def assign_extension(all_code: str, tags: List[str]) -> Tuple[str, str]: 14 | """Guess language and extension""" 15 | if not all_code: 16 | return "", "" 17 | 18 | result = guess_language_all_methods(code=all_code, tags=tags) 19 | if result: 20 | if result in FILE_EXTENSIONS: 21 | # take top extension option 22 | return FILE_EXTENSIONS[result][0], result 23 | return f".{result}", result 24 | 25 | return "", "" 26 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_npm_pyright.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code checks, but tool is a node app. 3 | """ 4 | 5 | from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple 6 | from navio_tasks.utils import inform 7 | 8 | 9 | def do_pyright() -> str: 10 | """ 11 | Execute pyright 12 | """ 13 | 14 | command = "pyright" 15 | if check_command_exists(command, throw_on_missing=False): 16 | # subprocess.check_call(("npm install -g pyright").split(" "), shell=True) 17 | inform( 18 | "You must install pyright before doing pyright checks: " 19 | "npm install -g pyright" 20 | ) 21 | command = prepinform_simple(command) 22 | command += "/**/*.py" 23 | execute(*(command.split(" "))) 24 | return "Pyright finished" 25 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_isort.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reorg imports. Constant 4 way battle between black, isort, pycharm and mscode 3 | """ 4 | 5 | from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple 6 | 7 | 8 | def do_isort() -> str: 9 | """Sort the imports to discover import order bugs and prevent import order bugs""" 10 | # This must run before black. black doesn't change import order but it wins 11 | # any arguments about formatting. 12 | # isort MUST be installed with pipx! It is not compatible with pylint in the same 13 | # venv. Maybe someday, but it just isn't worth the effort. 14 | 15 | check_command_exists("isort") 16 | command = "isort --profile black" 17 | command = prepinform_simple(command) 18 | execute(*(command.split(" "))) 19 | return "isort succeeded" 20 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/cli_mccabe.py: -------------------------------------------------------------------------------- 1 | """ 2 | Complexity reports 3 | """ 4 | 5 | import shlex 6 | 7 | from navio_tasks import settings as settings 8 | from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple 9 | from navio_tasks.settings import COMPLEXITY_CUT_OFF 10 | 11 | 12 | def do_mccabe() -> str: 13 | """ 14 | Complexity Checker 15 | """ 16 | 17 | check_command_exists("flake8") # yes, flake8, this is a plug in. 18 | # mccabe doesn't have a direct way to run it 19 | 20 | command_text = ( 21 | f"flake8 --max-complexity {COMPLEXITY_CUT_OFF} " 22 | f"--config {settings.CONFIG_FOLDER}/.flake8" 23 | ) 24 | command_text = prepinform_simple(command_text) 25 | command = shlex.split(command_text) 26 | execute(*command) 27 | return "mccabe succeeded" 28 | -------------------------------------------------------------------------------- /.config/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # This requirements file has been automatically generated from `Pipfile` with 3 | # `pipenv-to-requirements` 4 | # 5 | # 6 | # This has been done to maintain backward compatibility with tools and services 7 | # that do not support `Pipfile` yet. 8 | # 9 | # Do NOT edit it directly, use `pipenv install [-d]` to modify `Pipfile` and 10 | # `Pipfile.lock` and then regenerate `requirements*.txt`. 11 | ################################################################################ 12 | 13 | beautifulsoup4 14 | checksumdir 15 | dodgy==0.2.1 16 | gitpython 17 | ifaddr 18 | liccheck 19 | pebble 20 | psutil 21 | pyrankvote 22 | pytest 23 | pytest-cov 24 | pytest-timeout 25 | pytest-xdist 26 | python-dotenv 27 | requests 28 | sshtunnel 29 | tox 30 | vermin 31 | wheel 32 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_bandit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | """ 8 | 9 | from navio_tasks.cli_commands import check_command_exists, execute, prepinform_simple 10 | 11 | 12 | def do_bandit(is_shell_script_like: bool) -> str: 13 | """ 14 | Security Checks 15 | 16 | Generally returns a small number of problems to fix. 17 | """ 18 | if is_shell_script_like: 19 | return ( 20 | "Skipping bandit, this code is shell script-like so it has security" 21 | "issues on purpose." 22 | ) 23 | 24 | command = "bandit" 25 | check_command_exists(command) 26 | command = "bandit -r" 27 | command = prepinform_simple(command) 28 | execute(*(command.split(" "))) 29 | return "bandit succeeded" 30 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_django_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | """ 8 | 9 | import os 10 | 11 | from navio_tasks.cli_commands import config_pythonpath, execute_with_environment 12 | from navio_tasks.settings import PYTHON 13 | 14 | 15 | # Legacy code. Don't use nose. 16 | def do_django_tests() -> None: 17 | """ 18 | manage.py tests 19 | """ 20 | 21 | # check_command_exists("") 22 | if os.path.exists("manage.py"): 23 | do_django_tests_regardless() 24 | 25 | 26 | def do_django_tests_regardless() -> None: 27 | """ 28 | Extracted function to make easier to test this task 29 | """ 30 | command = f"{PYTHON} manage.py test -v 2" 31 | my_env = config_pythonpath() 32 | execute_with_environment(command, env=my_env) 33 | -------------------------------------------------------------------------------- /navio_tasks/dependency_commands/cli_safety.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | """ 8 | 9 | import subprocess 10 | 11 | from navio_tasks import settings as settings 12 | from navio_tasks.cli_commands import check_command_exists, execute 13 | 14 | 15 | def do_safety() -> str: 16 | """ 17 | Check free database for vulnerabilities in pinned libraries. 18 | """ 19 | requirements_file_name = f"{settings.CONFIG_FOLDER}/requirements_for_safety.txt" 20 | with open(requirements_file_name, "w+") as out: 21 | subprocess.run(["pip", "freeze"], stdout=out, stderr=out, check=True) 22 | check_command_exists("safety") 23 | # ignore 38414 until aws fixes awscli 24 | execute("safety", "check", "--ignore", "38414", "--file", requirements_file_name) 25 | return "Package safety checked" 26 | -------------------------------------------------------------------------------- /docs/TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | ## Release 1.0 goals 4 | - performance tuning 5 | - gather information on voter algorithms (do they ever tip elections) 6 | - packaging 7 | - a few more usage examples 8 | - implement safe json/yaml/xml parser (others?) 9 | - explicitly deal with "clones" (e.g. c is valid c++, javascript is valid TypeScript) 10 | 11 | ## Release 2.0 goals 12 | - popularity vote (i.e. boost guesses if the guess is a top 20 popular language) 13 | - refactor language data (name & alts, extension & alts, tags & alts) 14 | - improve tag based using StackOverflow data, e.g. numpy tag is probably python code 15 | - improve shebang code to cover more 16 | 17 | ## Release 3.0 goals 18 | - Classify code vs not-code, e.g. code of unknown language vs probably orindary english. 19 | - ref: https://softwareengineering.stackexchange.com/questions/87611/simple-method-for-reliably-detecting-code-in-text 20 | - Detect language of a collection of files, e.g. detect a python package (which could contain other files) 21 | -------------------------------------------------------------------------------- /navio_tasks/pure_reports/cli_gitchangelog.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extract commit comments from git to a report. Makes for a lousy CHANGELOG.md 3 | """ 4 | 5 | import shlex 6 | 7 | from navio_tasks.cli_commands import ( 8 | check_command_exists, 9 | config_pythonpath, 10 | execute_get_text, 11 | ) 12 | from navio_tasks.settings import VENV_SHELL 13 | from navio_tasks.utils import inform 14 | 15 | 16 | def do_gitchangelog() -> None: 17 | """ 18 | Extract commit comments from git to a report. Makes for a lousy CHANGELOG.md 19 | """ 20 | # TODO: this app has lots of features for cleaning up comments 21 | command_name = "gitchangelog" 22 | check_command_exists(command_name) 23 | 24 | command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ") 25 | inform(command_text) 26 | command = shlex.split(command_text) 27 | with open("ChangeLog", "w+") as change_log: 28 | result = execute_get_text(command, env=config_pythonpath()).replace("\r", "") 29 | change_log.write(result) 30 | -------------------------------------------------------------------------------- /navio_tasks/network.py: -------------------------------------------------------------------------------- 1 | """ 2 | Builds fail when they're not on VPN or can't reach remote servers 3 | """ 4 | 5 | import ifaddr 6 | import requests 7 | 8 | 9 | def check_public_ip(ipify: str = "https://api64.ipify.org/") -> str: 10 | """ 11 | Try to get the elastic IP for this machine 12 | """ 13 | # BUG: When this fails, it doesn't fail fast. 14 | 15 | # pylint: disable=bare-except,broad-except 16 | # noinspection PyBroadException 17 | try: 18 | # https://api.ipify.org # fails on VPN. 19 | # pylint: disable=invalid-name 20 | ip = requests.get(ipify).text 21 | return ip 22 | except BaseException: # noqa 23 | return "" 24 | 25 | 26 | def is_known_network(prefix: str) -> bool: 27 | """Are we on a known network""" 28 | 29 | adapters = ifaddr.get_adapters() 30 | for adapter in adapters: 31 | # pylint: disable=invalid-name 32 | for ip in adapter.ips: 33 | if str(ip.ip).startswith(prefix): 34 | return True 35 | return False 36 | -------------------------------------------------------------------------------- /whats_that_code/__main__.py: -------------------------------------------------------------------------------- 1 | # noinspection PyPep8 2 | """whats_that_code/identify sourcode 3 | Usage: 4 | whats_that_code 5 | whats_that_code 6 | so_pip (-h | --help) 7 | so_pip --version 8 | 9 | Options: 10 | -h --help Show this screen. 11 | -v --version Show version. 12 | -c --code= Source code. 13 | -f --file= Path to file 14 | --verbose Show logging 15 | --quiet No informational logging 16 | 17 | """ 18 | import logging 19 | import sys 20 | 21 | import docopt 22 | 23 | from whats_that_code import _version as meta 24 | 25 | # Do these need to stick around? 26 | LOGGERS = [] 27 | 28 | LOGGER = logging.getLogger(__name__) 29 | 30 | 31 | def main() -> int: 32 | """Get the args object from command parameters""" 33 | arguments = docopt.docopt(__doc__, version=f"so_pip {meta.__version__}") 34 | LOGGER.debug(arguments) 35 | return 0 36 | 37 | 38 | if __name__ == "__main__": 39 | sys.exit(main()) 40 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_go_gitleaks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implemented in go. 3 | 4 | Hangs on my workstation. 5 | """ 6 | 7 | import os 8 | import shlex 9 | import subprocess 10 | 11 | from navio_tasks.settings import PROJECT_NAME 12 | from navio_tasks.utils import inform 13 | 14 | 15 | def run_gitleaks() -> None: 16 | """ 17 | Run the gitleaks command. 18 | 19 | Depends on go! 20 | """ 21 | # git remote get-url --all origin 22 | # So far nothing works... as if current repo is corrupt 23 | cwd = os.getcwd() 24 | command_text = "gitleaks --repo-path={} --report=/tmp/{}.csv".format( 25 | cwd, PROJECT_NAME 26 | ).strip() 27 | inform(command_text) 28 | command = shlex.split(command_text) 29 | _ = subprocess.run( 30 | command, 31 | capture_output=True, 32 | env={ 33 | **os.environ, 34 | "GOPATH": os.path.expandvars("$HOME/gocode"), 35 | "PATH": os.path.expandvars("$PATH/$GOPATH/bin"), 36 | }, 37 | # shell=False, # keep false if possible. 38 | check=True, 39 | ) 40 | -------------------------------------------------------------------------------- /navio_tasks/output.py: -------------------------------------------------------------------------------- 1 | """ 2 | Console output related 3 | """ 4 | 5 | import subprocess 6 | import sys 7 | 8 | from navio_tasks import settings as settings 9 | from navio_tasks.cli_commands import check_command_exists 10 | from navio_tasks.system_info import check_is_aws 11 | from navio_tasks.utils import inform 12 | 13 | 14 | def say_and_exit(message: str, source: str) -> None: 15 | """ 16 | Audibly notify the developer that the build is 17 | done so that long builds wouldn't cause an attention problem 18 | """ 19 | inform(f"{source}:{message}") 20 | if ( 21 | settings.SPEAK_WHEN_BUILD_FAILS 22 | and settings.IS_INTERACTIVE 23 | and not check_is_aws() 24 | ): 25 | # TODO: check a profile option or something. 26 | if check_command_exists("say", throw_on_missing=False, exit_on_missing=False): 27 | subprocess.call(["say", message]) 28 | elif check_command_exists( 29 | "wsay.exe", throw_on_missing=False, exit_on_missing=False 30 | ): 31 | subprocess.call(["wsay", message]) 32 | sys.exit(-1) 33 | -------------------------------------------------------------------------------- /navio_tasks/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Tools that check, fix, package, etc. 4 | - [Commands](Commands) are the wrappers around specific cli tools 5 | - [lib_tools](lib_tools) are wrappers around tools w/o a cli interface 6 | 7 | Things for build scripts 8 | - [Build state](build_state.py) keeps track of when source code last changed relative to last 9 | successful run 10 | - [Clean](clean.py) deletes old outputs so that you don't get a mix of new and old 11 | - [cli_commands](cli_commands.py) are mostly wrappers around subprocess (shell) calls 12 | - [settings](settings.py) Per project variations 13 | 14 | Features 15 | -------- 16 | - Policies 17 | - code size cut offs (some tools don't make sense on tiny code bases) 18 | - strictness 19 | - environment adaption 20 | - mac/window 21 | - container/not container 22 | - workstation/build server/cloud machine 23 | - VPN/Org Network vs No network vs Home network 24 | - abstracting away the interface 25 | - every tool has its own way of specifying the files/folder to act on 26 | - environment variable input vs switches 27 | - cli interface vs importable python interface 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Matthew Martin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /whats_that_code/regex_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fast detect python. 3 | 4 | https://github.com/TomCrypto/Codex/blob/master/codex.py 5 | """ 6 | 7 | from typing import Dict, List 8 | 9 | from whats_that_code.codex_markers import MARKERS 10 | 11 | 12 | def language_by_regex_features( 13 | text: str, just_code_is_valid: bool = False 14 | ) -> List[str]: 15 | """Count features that signal python, multiples 16 | worth extra only for 2nd. 17 | just_code_is_valid - if True, "code" is a valid guess. 18 | """ 19 | 20 | guesses: Dict[str, int] = {} 21 | for language, finders in MARKERS.items(): 22 | if not just_code_is_valid and language == "code": 23 | continue 24 | for finder in finders: 25 | found = finder.findall(text) 26 | if len(found) > 0: 27 | if language in guesses: 28 | guesses[language] += len(found) 29 | else: 30 | guesses[language] = len(found) 31 | 32 | results = [ 33 | language 34 | for language, score in sorted(guesses.items(), key=lambda item: -item[1]) 35 | ] 36 | return results 37 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_pyupgrade.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reformat code to use py3 patterns when possible 3 | """ 4 | 5 | import glob 6 | 7 | from navio_tasks.cli_commands import check_command_exists, execute 8 | from navio_tasks.settings import PROJECT_NAME, VENV_SHELL 9 | from navio_tasks.utils import inform 10 | 11 | 12 | def do_pyupgrade(is_interactive: bool, minimum_python: str) -> str: 13 | """Update syntax to most recent variety.""" 14 | if not is_interactive: 15 | inform("Not an interactive session, skipping pyupgrade wihch changes files.") 16 | return "Skipping" 17 | command = "pyupgrade" 18 | check_command_exists(command) 19 | 20 | all_files = " ".join( 21 | f for f in glob.glob(f"{PROJECT_NAME}/**/*.py", recursive=True) 22 | ) 23 | 24 | # as of 2021, still doesn't appear to support recursive globs natively. 25 | command = ( 26 | f"{VENV_SHELL} pyupgrade " 27 | f"--{minimum_python}-plus " 28 | f"--exit-zero-even-if-changed {all_files}".strip().replace(" ", " ") 29 | ) 30 | inform(command) 31 | execute(*(command.split(" "))) 32 | return f"{command} succeeded" 33 | -------------------------------------------------------------------------------- /scripts/get_most_popular.py: -------------------------------------------------------------------------------- 1 | """ 2 | Top programming languages this year 3 | """ 4 | 5 | import requests 6 | from bs4 import BeautifulSoup 7 | 8 | # result =requests.get("https://pypl.github.io/PYPL.html?country=US") 9 | # 10 | # soup = BeautifulSoup(result.text, features="html.parser") 11 | # table = soup.find_all("") 12 | # print() 13 | if __name__ == "__main__": 14 | ranks = """1 Python 15 | 2 Java 16 | 3 JavaScript 17 | 4 C# 18 | 5 C/C++ 19 | 6 PHP 20 | 7 R 21 | 8 Objective-C 22 | 9 Swift 23 | 10 TypeScript 24 | 11 Matlab 25 | 12 Kotlin 26 | 13 Go 27 | 14 VBA 28 | 15 Ruby 29 | 16 Rust 30 | 17 Scala 31 | 18 Visual Basic 32 | 19 Lua 33 | 20 Ada 34 | 21 Dart 35 | 22 Abap 36 | 23 Perl 37 | 24 Julia 38 | 25 Groovy 39 | 26 Cobol 40 | 27 Haskell 41 | 28 Delphi""" 42 | data = {} 43 | data_list = [] 44 | for row in ranks.split("\n"): 45 | rank, _, language = row.split("\t") 46 | data[language.lower()] = rank 47 | data_list.append(language.lower()) 48 | print(data) 49 | print(data_list) 50 | -------------------------------------------------------------------------------- /.config/requirements_for_safety.txt: -------------------------------------------------------------------------------- 1 | apipkg==1.5 2 | appdirs==1.4.4 3 | atomicwrites==1.4.0 4 | attrs==20.3.0 5 | bcrypt==3.2.0 6 | beautifulsoup4==4.9.3 7 | certifi==2020.12.5 8 | cffi==1.14.4 9 | chardet==4.0.0 10 | checksumdir==1.2.0 11 | colorama==0.4.4 12 | configparser==5.0.1 13 | coverage==5.3.1 14 | cryptography==3.3.1 15 | defusedxml==0.6.0 16 | distlib==0.3.1 17 | dodgy==0.2.1 18 | execnet==1.7.1 19 | filelock==3.0.12 20 | gitdb==4.0.5 21 | GitPython==3.1.12 22 | idna==2.10 23 | ifaddr==0.1.7 24 | iniconfig==1.1.1 25 | liccheck==0.4.9 26 | navio-builder-win==0.1.57 27 | packaging==20.8 28 | paramiko==2.7.2 29 | Pebble==4.6.0 30 | pluggy==0.13.1 31 | psutil==5.8.0 32 | py==1.10.0 33 | pycparser==2.20 34 | Pygments==2.7.4 35 | PyNaCl==1.4.0 36 | pyparsing==2.4.7 37 | pyrankvote==2.0.2 38 | pytest==6.2.1 39 | pytest-cov==2.10.1 40 | pytest-forked==1.3.0 41 | pytest-timeout==1.4.2 42 | pytest-xdist==2.2.0 43 | python-dotenv==0.15.0 44 | requests==2.25.1 45 | semantic-version==2.8.5 46 | sh==1.14.1 47 | six==1.15.0 48 | smmap==3.0.4 49 | soupsieve==2.1 50 | sshtunnel==0.4.0 51 | tabulate==0.8.7 52 | toml==0.10.2 53 | tox==3.21.1 54 | urllib3==1.26.2 55 | vermin==1.1.0 56 | virtualenv==20.3.1 57 | -------------------------------------------------------------------------------- /whats_that_code/extension_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | Guess by extension 3 | """ 4 | 5 | import os 6 | from typing import List 7 | 8 | from whats_that_code.known_languages import FILE_EXTENSIONS 9 | 10 | 11 | def guess_by_extension(file_name: str = "", text: str = "") -> List[str]: 12 | """ 13 | Guess by extensions 14 | """ 15 | if not file_name and not text: 16 | return [] 17 | _, extension_part = os.path.splitext(file_name) 18 | 19 | if not extension_part: 20 | return [] 21 | 22 | guesses = set() 23 | extensions = [] 24 | if file_name: 25 | # TODO: use 26 | extensions.append(extension_part) 27 | if text: 28 | words = [_ for _ in text.split(" ") if "." in _] 29 | for word in words: 30 | if word.endswith("."): 31 | continue 32 | _, extension = os.path.splitext(word) 33 | if 1 < len(extension) < 5: 34 | extensions.append(extension) 35 | 36 | for key, value in FILE_EXTENSIONS.items(): 37 | for extension in extensions: 38 | for alternative in value: 39 | if extension == alternative: 40 | guesses.add(key) 41 | 42 | return list(guesses) 43 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_compile_py.py: -------------------------------------------------------------------------------- 1 | """ 2 | See if python compiles. 3 | 4 | TODO: Maybe implement vermin, another compilation checker. 5 | """ 6 | 7 | import shlex 8 | 9 | from navio_tasks.cli_commands import ( 10 | config_pythonpath, 11 | execute_get_text, 12 | prepinform_simple, 13 | ) 14 | from navio_tasks.utils import inform 15 | 16 | 17 | def do_compile_py(python: str) -> str: 18 | """ 19 | Catch only the worst syntax errors in the currently python version 20 | """ 21 | command_text = f"{python} -m compileall" 22 | command_text = prepinform_simple(command_text) 23 | command = shlex.split(command_text) 24 | result = execute_get_text(command, env=config_pythonpath()) 25 | for line in result.split("\n"): 26 | if line and (line.startswith("Compiling") or line.startswith("Listing")): 27 | pass 28 | else: 29 | inform(line) 30 | return "compileall succeeded" 31 | 32 | 33 | # Vermin sample output 34 | # Tip: You're using potentially backported modules: argparse, configparser, typing 35 | # If so, try using the following for better results: --backport argparse 36 | # --backport configparser --backport typing 37 | # 38 | # Minimum required versions: 3.6 39 | # Incompatible versions: 2 40 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_tox.py: -------------------------------------------------------------------------------- 1 | """ 2 | See if everything works with python 3.8 and upcoming libraries 3 | """ 4 | 5 | import shlex 6 | 7 | from navio_tasks import settings as settings 8 | from navio_tasks.cli_commands import check_command_exists, execute 9 | from navio_tasks.settings import VENV_SHELL 10 | from navio_tasks.utils import inform 11 | 12 | 13 | def do_tox() -> str: 14 | """ 15 | See if everything works with python 3.8 and upcoming libraries 16 | """ 17 | # If tox fails the build with 3.8 or some future library, that means 18 | # we can't migrate to 3.8 yet, or that we should stay with currently pinned 19 | # libraries. We should fail the overall build. 20 | # 21 | # Because we control our python version we don't have to support cross ver 22 | # compatibility, i.e. we are not supporting 2.7 & 3.x! 23 | if settings.TOX_ACTIVE: 24 | # this happens when testing the build script itself. 25 | return "tox already, not nesting" 26 | command_name = "tox" 27 | check_command_exists(command_name) 28 | 29 | command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ") 30 | inform(command_text) 31 | command = shlex.split(command_text) 32 | execute(*command) 33 | return "tox succeeded" 34 | -------------------------------------------------------------------------------- /whats_that_code/tag_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | Guess by SO Tag 3 | """ 4 | 5 | from typing import List 6 | 7 | from whats_that_code.known_languages import FILE_EXTENSIONS 8 | from whats_that_code.tags_data import RELATED_TAGS 9 | 10 | 11 | def match_tag_to_languages(tags: List[str]) -> List[str]: 12 | """Guess language by tag""" 13 | if not tags: 14 | return [] 15 | # eg. tagged python, then guess python 16 | strong_guesses = set() 17 | for tag in (_.lower() for _ in tags): 18 | if not tag: 19 | continue 20 | if tag in FILE_EXTENSIONS: 21 | strong_guesses.add(tag) 22 | 23 | # e.g. tagged, numpy, which is commonly related to python 24 | weak_guesses = set() 25 | for tag in tags: 26 | for key, value in RELATED_TAGS.items(): 27 | if tag in value[0:3]: 28 | weak_guesses.add(key) 29 | 30 | new_weak_guesses = [] 31 | for guess in weak_guesses: 32 | if guess not in strong_guesses: 33 | new_weak_guesses.append(guess) 34 | return list(strong_guesses) + list(new_weak_guesses) 35 | 36 | 37 | # Old algo 38 | # def match_tag_to_language(tags: List[str]) -> Optional[Tuple[str, str]]: 39 | # """Guess language by tag""" 40 | # for tag in (_.lower() for _ in tags): 41 | # if not tag: 42 | # continue 43 | # if tag in FILE_EXTENSIONS: 44 | # return FILE_EXTENSIONS[tag], tag 45 | # return None 46 | -------------------------------------------------------------------------------- /navio_tasks/packaging_commands/cli_twine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload to pypi. Pypi is public, don't do that unless you mean to. 3 | """ 4 | 5 | import os 6 | 7 | from navio_tasks.cli_commands import check_command_exists, execute 8 | 9 | 10 | def do_upload_package() -> None: 11 | """ 12 | Send to private package repo 13 | """ 14 | # devpi use http://localhost:3141 15 | # login with root... 16 | # devpi login root --password= 17 | # get indexes 18 | # devpi use -l 19 | # make an index 20 | # devpi index -c dev bases=root/pypi 21 | # devpi use root/dev 22 | 23 | # Must register (may go away with newer version of devpi), must be 1 file! 24 | # twine register --config-file .pypirc -r devpi-root -u root 25 | # -p PASSWORD dist/search_service-0.1.0.zip 26 | # can be all files! 27 | # twine upload --config-file .pypirc -r devpi-root -u root -p PASSWORD dist/* 28 | 29 | # which is installable using... 30 | # pip install search-service --index-url=http://localhost:3141/root/dev/ 31 | 32 | check_command_exists("devpi") 33 | password = os.environ["DEVPI_PASSWORD"] 34 | any_zip = [file for file in os.listdir("dist") if file.endswith(".zip")][0] 35 | register_command = ( 36 | "twine register --config-file .pypirc -r devpi-root -u root" 37 | f" -p {password} dist/{any_zip}" 38 | ) 39 | upload_command = ( 40 | "twine upload --config-file .pypirc -r devpi-root " 41 | f"-u root -p {password} dist/*" 42 | ) 43 | 44 | execute(*(register_command.strip().split(" "))) 45 | execute(*(upload_command.strip().split(" "))) 46 | -------------------------------------------------------------------------------- /navio_tasks/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code from pynt until I can phase it out. 3 | """ 4 | 5 | import contextlib 6 | import functools 7 | import os 8 | import sys 9 | from subprocess import CalledProcessError, check_call 10 | from typing import Any, Iterable 11 | 12 | 13 | __license__ = "MIT License" 14 | __contact__ = "http://rags.github.com/pynt-contrib/" 15 | 16 | # pylint: disable=redefined-builtin 17 | inform = functools.partial(print, flush=True) # no qa 18 | 19 | 20 | @contextlib.contextmanager 21 | def safe_cd(path: str) -> Any: 22 | """ 23 | Changes to a directory, yields, and changes back. 24 | Additionally any error will also change the directory back. 25 | 26 | Usage: 27 | # >>> with safe_cd('some/repo'): 28 | # ... execute('git status') 29 | """ 30 | starting_directory = os.getcwd() 31 | try: 32 | os.chdir(path) 33 | yield 34 | finally: 35 | os.chdir(starting_directory) 36 | 37 | 38 | def execute(script: str, *args: Iterable[str]) -> int: 39 | """ 40 | Executes a command through the shell. Spaces should breakup the args. 41 | Usage: execute('grep', 'TODO', '*') 42 | """ 43 | 44 | popen_args = [script] + list(args) 45 | # pylint: disable=broad-except 46 | try: 47 | return check_call(popen_args, shell=False) # noqa 48 | except CalledProcessError as ex: 49 | inform(ex) 50 | sys.exit(ex.returncode) 51 | except Exception as ex: 52 | inform(f"Error: {ex} with script: {script} and args {args}") 53 | sys.exit(1) 54 | 55 | 56 | # 57 | # def inform(*args: Any) -> None: 58 | # print(args) 59 | 60 | # keep this one 61 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/cli_vulture.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dead Code 3 | """ 4 | 5 | import shlex 6 | import subprocess 7 | import sys 8 | 9 | from navio_tasks.cli_commands import check_command_exists, config_pythonpath 10 | from navio_tasks.output import say_and_exit 11 | from navio_tasks.pure_reports.cli_pygount import total_loc 12 | from navio_tasks.settings import ( 13 | MAXIMUM_DEAD_CODE, 14 | PROBLEMS_FOLDER, 15 | PROJECT_NAME, 16 | SMALL_CODE_BASE_CUTOFF, 17 | VENV_SHELL, 18 | ) 19 | from navio_tasks.utils import inform 20 | 21 | 22 | def do_vulture() -> str: 23 | """ 24 | This also finds code you are working on today! 25 | """ 26 | 27 | check_command_exists("vulture") 28 | 29 | # TODO: check if whitelist.py exists? 30 | command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py" 31 | command_text = command_text.strip().replace(" ", " ") 32 | inform(command_text) 33 | command = shlex.split(command_text) 34 | 35 | output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt" 36 | with open(output_file_name, "w") as outfile: 37 | env = config_pythonpath() 38 | subprocess.call(command, stdout=outfile, env=env) 39 | 40 | if total_loc() > SMALL_CODE_BASE_CUTOFF: 41 | cutoff = MAXIMUM_DEAD_CODE 42 | else: 43 | cutoff = 0 44 | 45 | with open(output_file_name) as file_handle: 46 | num_lines = sum(1 for line in file_handle if line) 47 | if num_lines > cutoff: 48 | say_and_exit( 49 | f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture" 50 | ) 51 | sys.exit(-1) 52 | return "dead-code (vulture) succeeded" 53 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_jiggle_version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Increase version number, but only the last number in a semantic version triplet 3 | """ 4 | 5 | import shlex 6 | 7 | from git import InvalidGitRepositoryError, Repo 8 | 9 | from navio_tasks.cli_commands import check_command_exists, execute 10 | from navio_tasks.settings import PROJECT_NAME 11 | from navio_tasks.utils import inform 12 | 13 | 14 | def do_jiggle_version( 15 | is_interactive: bool, 16 | target_branch: str = "", 17 | increase_version_on_all_branches: bool = False, 18 | ) -> str: 19 | """ 20 | Increase version number, but only the last number in a semantic version triplet 21 | """ 22 | if not is_interactive: 23 | inform( 24 | "Not an interactive session, skipping jiggle_version, which changes files." 25 | ) 26 | return "Skipping" 27 | # rorepo is a Repo instance pointing to the git-python repository. 28 | # For all you know, the first argument to Repo is a path to the repository 29 | # you want to work with 30 | try: 31 | repo = Repo(".") 32 | active_branch = str(repo.active_branch) 33 | except InvalidGitRepositoryError: 34 | inform("Can't detect what branch we are on. Is this a git repo?") 35 | active_branch = "don't know what branch we are on" 36 | 37 | if active_branch == target_branch or increase_version_on_all_branches: 38 | check_command_exists("jiggle_version") 39 | command = f"jiggle_version here --module={PROJECT_NAME}" 40 | parts = shlex.split(command) 41 | execute(*parts) 42 | else: 43 | inform("Not master branch, not incrementing version") 44 | return "ok" 45 | -------------------------------------------------------------------------------- /navio_tasks/dependency_commands/cli_pip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Register scripts 3 | """ 4 | 5 | import os 6 | import shlex 7 | import sys 8 | 9 | from navio_tasks.cli_commands import ( 10 | check_command_exists, 11 | config_pythonpath, 12 | execute, 13 | execute_with_environment, 14 | ) 15 | from navio_tasks.settings import PIPENV_ACTIVE, PROJECT_NAME, VENV_SHELL 16 | from navio_tasks.utils import inform 17 | 18 | 19 | def do_register_scripts() -> None: 20 | """ 21 | Without this console_scripts in the entrypoints section of setup.py aren't 22 | available 23 | :return: 24 | """ 25 | check_command_exists("pip") 26 | 27 | # This doesn't work, it can't tell if "install -e ." has already run 28 | if dist_is_editable(): 29 | inform("console_scripts already registered") 30 | return 31 | # install in "editable" mode 32 | command_text = f"{VENV_SHELL} pip install -e ." 33 | inform(command_text) 34 | command_text = command_text.strip().replace(" ", " ") 35 | command = shlex.split(command_text) 36 | execute(*command) 37 | 38 | 39 | def dist_is_editable() -> bool: 40 | """Is distribution an editable install?""" 41 | for path_item in sys.path: 42 | egg_link = os.path.join(path_item, PROJECT_NAME + ".egg-link") 43 | if os.path.isfile(egg_link): 44 | return True 45 | return False 46 | 47 | 48 | def do_pip_check() -> str: 49 | """ 50 | Call as normal function 51 | """ 52 | execute("pip", "check") 53 | environment = config_pythonpath() 54 | environment["PIPENV_PYUP_API_KEY"] = "" 55 | if PIPENV_ACTIVE: 56 | # ignore 38414 until aws fixes awscli 57 | execute_with_environment("pipenv check --ignore 38414", environment) 58 | return "Pip(env) check run" 59 | -------------------------------------------------------------------------------- /navio_tasks/non_py_commands/cli_openapi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Does swagger/openapi file parse 3 | """ 4 | 5 | import os 6 | import shlex 7 | import sys 8 | 9 | from navio_tasks.cli_commands import config_pythonpath, execute, execute_get_text 10 | from navio_tasks.output import say_and_exit 11 | from navio_tasks.settings import IS_GITLAB, IS_JENKINS, PROJECT_NAME, VENV_SHELL 12 | from navio_tasks.utils import inform 13 | 14 | 15 | def do_openapi_check() -> None: 16 | """ 17 | Does swagger/openapi file parse 18 | """ 19 | if not os.path.exists(f"{PROJECT_NAME}/api.yaml"): 20 | inform("No api.yaml file, assuming this is not a microservice") 21 | # TODO: should be able to check all y?ml files and look at header. 22 | return 23 | 24 | command_text = ( 25 | f"{VENV_SHELL} " 26 | "openapi-spec-validator" 27 | f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " ") 28 | ) 29 | inform(command_text) 30 | command = shlex.split(command_text) 31 | execute(*command) 32 | 33 | if IS_JENKINS or IS_GITLAB: 34 | inform("Jenkins/Gitlab and apistar don't work together, skipping") 35 | return 36 | 37 | command_text = ( 38 | f"{VENV_SHELL} apistar validate " 39 | f"--path {PROJECT_NAME}/api.yaml " 40 | "--format openapi " 41 | "--encoding yaml".strip().replace(" ", " ") 42 | ) 43 | inform(command_text) 44 | # subprocess.check_call(command.split(" "), shell=False) 45 | command = shlex.split(command_text) 46 | result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) 47 | if "OK" not in result and "2713" not in result and "✓" not in result: 48 | inform(result) 49 | say_and_exit("apistar didn't like this", "apistar") 50 | sys.exit(-1) 51 | -------------------------------------------------------------------------------- /navio_tasks/dependency_commands/cli_pin_dependencies.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pin deps 3 | """ 4 | 5 | import os 6 | 7 | from navio_tasks import settings as settings 8 | from navio_tasks.cli_commands import check_command_exists, execute 9 | from navio_tasks.settings import VENV_SHELL 10 | from navio_tasks.utils import inform 11 | 12 | 13 | def convert_pipenv_to_requirements(pipenv: bool) -> None: 14 | """ 15 | Create requirement*.txt 16 | """ 17 | if not pipenv: 18 | raise TypeError( 19 | "Can't pin dependencies this way, we are only converting " 20 | "pipfile to requirements.txt" 21 | ) 22 | check_command_exists("pipenv_to_requirements") 23 | 24 | execute( 25 | *( 26 | f"{VENV_SHELL} pipenv_to_requirements " 27 | f"--dev-output {settings.CONFIG_FOLDER}/requirements-dev.txt " 28 | f"--output {settings.CONFIG_FOLDER}/requirements.txt".strip().split(" ") 29 | ) 30 | ) 31 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): 32 | inform( 33 | "Warning: no requirements.txt found, assuming it is because there are" 34 | "no external dependencies yet" 35 | ) 36 | else: 37 | with open(f"{settings.CONFIG_FOLDER}/requirements.txt", "r+") as file: 38 | lines = file.readlines() 39 | file.seek(0) 40 | for line in lines: 41 | if line.find("-e .") == -1: 42 | file.write(line) 43 | file.truncate() 44 | 45 | with open(f"{settings.CONFIG_FOLDER}/requirements-dev.txt", "r+") as file: 46 | lines = file.readlines() 47 | file.seek(0) 48 | for line in lines: 49 | if line.find("-e .") == -1: 50 | file.write(line) 51 | file.truncate() 52 | -------------------------------------------------------------------------------- /navio_tasks/dependency_commands/cli_installs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Install dependencies so that build has a chance of succeeding. 3 | """ 4 | 5 | import os 6 | import shlex 7 | 8 | from navio_tasks.cli_commands import check_command_exists, execute 9 | from navio_tasks.utils import inform 10 | 11 | 12 | def do_dependency_installs(pipenv: bool, poetry: bool, pip: bool) -> None: 13 | """Catch up with missing deps""" 14 | if pipenv: 15 | check_command_exists("pipenv") 16 | command_text = "pipenv install --dev --skip-lock" 17 | inform(command_text) 18 | command = shlex.split(command_text) 19 | execute(*command) 20 | elif poetry: 21 | check_command_exists("poetry") 22 | command_text = "poetry install" 23 | inform(command_text) 24 | command = shlex.split(command_text) 25 | execute(*command) 26 | elif pip: 27 | # TODO: move code to deprecated section? 28 | # TODO: Check for poetry. 29 | if os.path.exists("Pipfile"): 30 | raise TypeError("Found Pipfile, settings imply we aren't using Pipenv.") 31 | if os.path.exists("requirements.txt"): 32 | command_text = "pip install -r requirements.txt" 33 | inform(command_text) 34 | command = shlex.split(command_text) 35 | execute(*command) 36 | else: 37 | inform("no requirements.txt file yet, can't install dependencies") 38 | 39 | if os.path.exists("requirements-dev.txt"): 40 | command_text = "pip install -r requirements-dev.txt" 41 | inform(command_text) 42 | command = shlex.split(command_text) 43 | execute(*command) 44 | else: 45 | inform("no requirements-dev.txt file yet, can't install dependencies") 46 | else: 47 | inform("VENV not previously activated, won't attempt to catch up on installs") 48 | -------------------------------------------------------------------------------- /whats_that_code/pygments_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | Guess with pygments which can have an accuracy of 10% on SO questions (seems low?) 3 | """ 4 | 5 | from typing import List 6 | 7 | from pygments import lexers 8 | from pygments.lexers import LEXERS 9 | 10 | from whats_that_code.known_languages import FILE_EXTENSIONS 11 | 12 | KNOWN_PYGMENTS = {} 13 | 14 | if not KNOWN_PYGMENTS: 15 | 16 | def init(): 17 | """Create entries""" 18 | for key, value in LEXERS.items(): 19 | # print(f' "{key.replace("Lexer","").lower()}", {list(value[3])},') 20 | KNOWN_PYGMENTS[key.replace("Lexer", "").lower()] = list(value[3]) 21 | 22 | init() 23 | 24 | 25 | def language_by_pygments(code: str) -> List[str]: 26 | """Guess by parsing""" 27 | if not code or not code.strip(): 28 | return [] 29 | guess = lexers.guess_lexer(code) 30 | 31 | if guess.name not in FILE_EXTENSIONS: 32 | if hasattr(guess, "language_lexer"): 33 | lexer_name = guess.language_lexer.name 34 | else: 35 | lexer_name = guess.name 36 | lexer_name = str(lexer_name).lower() 37 | if lexer_name in KNOWN_PYGMENTS: 38 | FILE_EXTENSIONS[guess.name] = KNOWN_PYGMENTS[lexer_name] 39 | else: 40 | pass 41 | # print("What pygment thing is this: " + lexer_name) 42 | 43 | # pygments confuses java & python 44 | if guess.name.lower().startswith("python"): 45 | return [] # ["python"] 46 | # pygments confuses gdscript and javascript 47 | if guess.name.lower().startswith("gdscript"): 48 | return [] 49 | if "mime" in guess.name.lower(): 50 | return [] 51 | 52 | # xml, html, php ...some are subsets of others 53 | if guess.name.lower() == "xml+php": 54 | # prefer guessing languages, use xml parser to detect xml 55 | return ["php"] 56 | return [guess.name.lower()] 57 | -------------------------------------------------------------------------------- /whats_that_code/shebang_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | Guess a language based on shebang 3 | """ 4 | 5 | # http://dcjtech.info/topic/list-of-shebang-interpreter-directives/ 6 | from typing import List 7 | 8 | from whats_that_code.known_languages import FILE_EXTENSIONS 9 | 10 | SHEBANGS = { 11 | "#!/usr/bin/python": "python", 12 | "#!/usr/bin/env python": "python", 13 | "#!/usr/bin/env python3": "python", 14 | "#!/bin/ash": "ash", 15 | "#!/usr/bin/awk": ".awk", 16 | "#!/bin/bash": "bash", 17 | "#!/usr/bin/env bash": "bash", 18 | "#!/bin/busybox sh": "bash", 19 | # "#!/bin/csh": ".csh", # right extension? 20 | "#!/usr/local/bin/groovy": "groovy", 21 | "#!/usr/bin/env groovy": "groovy", 22 | "#!/usr/bin/env jsc": "javascript", 23 | "#!/usr/bin/env node": "javascript", 24 | "#!/usr/bin/env rhino": "javascript", 25 | "#!/usr/local/bin/sbcl --script": "lisp", 26 | "#!/usr/bin/env lua": "lua", 27 | "#!/usr/bin/lua": "lua", 28 | "#!/usr/bin/make -f": "make", 29 | "#!/usr/bin/env perl": "perl", 30 | "#!/usr/bin/perl": "perl", 31 | "#!/usr/bin/perl -T": "perl", 32 | "#!/usr/bin/php": "php", 33 | "#!/usr/bin/env php": "php", 34 | "#!/usr/bin/env ruby": "ruby", 35 | "#!/usr/bin/ruby": "ruby", 36 | "#!/bin/sed -f": "sed", 37 | "#!/usr/bin/sed -f": "sed", 38 | "#!/usr/bin/env sed": "sed", 39 | # "#!/bin/sh": "bash", 40 | # "#!/usr/xpg4/bin/sh": ".sh", 41 | # "#!/bin/tcsh": ".tcsh", 42 | } 43 | 44 | 45 | def language_by_shebang(test: str) -> List[str]: 46 | """Identify by shebang""" 47 | possibles = set() 48 | for key, value in SHEBANGS.items(): 49 | if key in test: 50 | possibles.add(value) 51 | if key.strip("#!/") in test: 52 | possibles.add(value) 53 | 54 | for possible in possibles: 55 | if possible not in FILE_EXTENSIONS: 56 | raise TypeError() 57 | return list(possibles) 58 | -------------------------------------------------------------------------------- /docs/prior_art.md: -------------------------------------------------------------------------------- 1 | # Prior Art 2 | Language detection is divided between libraries that are classifiers, and classifiers by accident, such as 3 | code highlighters or line counters. 4 | 5 | Python 6 | - [Guesslang](https://pypi.org/project/guesslang/) - python and tensorflow driven solution. 7 | - [Deep Learning](https://github.com/tunafield/deep-learning-lang-detection) - keras and tensor flow. 8 | - [Programming Language Detection](https://github.com/batogov/programming-language-detection) - sklearn based 9 | - [Codex](https://github.com/TomCrypto/Codex) - sklearn based 10 | - [Code Sleuth](https://github.com/scivision/code-sleuth) - Currently only detects language by project file? 11 | 12 | Not Python 13 | - [Language Detection EL](https://github.com/andreasjansson/language-detection.el) - Lisp. Editor plugin 14 | - [Github's Linguist](https://github.com/github/linguist) Ruby. Classifies code in repos. 15 | - [Source Classifier](https://github.com/chrislo/sourceclassifier) Ruby. 16 | - [Code Classifier](https://github.com/bertyhell/CodeClassifier) C#. 17 | 18 | Highlight/Formatter/Line Counter/Decompilation 19 | - [highlight.js](https://github.com/highlightjs/highlight.js). Javascript 20 | - [code-prettify](https://github.com/googlearchive/code-prettify). Javascript. 21 | - [pygments](https://pygments.org/docs/api/#pygments.lexers.guess_lexer). Python 22 | - [OhCount](https://github.com/blackducksoftware/ohcount) Ruby. 23 | - [IDAPro](https://www.hex-rays.com/products/ida/) Commercial decompiler. Identifies langauge of binary code. 24 | 25 | Research Papers, possibly with no public source 26 | - [Predicting SO](https://www.researchgate.net/publication/338132359_SCC_Predicting_the_Programming_Language_of_Questions_and_Snippets_of_StackOverflow) 27 | - [Source code for above paper](https://github.com/Kamel773/SourceCodeClassification) 28 | - [Github's OctoLingua](https://github.com/MankaranSingh/GSoC-2020/blob/master/README.md) Closed source classifier? 29 | -------------------------------------------------------------------------------- /navio_tasks/system_info.py: -------------------------------------------------------------------------------- 1 | """ 2 | Infering what machine we are on. 3 | """ 4 | 5 | import os 6 | import platform 7 | import re 8 | import socket 9 | import git 10 | import psutil 11 | 12 | from navio_tasks.utils import inform 13 | 14 | 15 | def is_git_repo(path: str) -> bool: 16 | """ 17 | Are we in a git repo if not, several tools don't make sense to run 18 | https://stackoverflow.com/a/39956572/33264 19 | """ 20 | try: 21 | _ = git.Repo(path).git_dir 22 | return True 23 | except git.exc.InvalidGitRepositoryError: 24 | return False 25 | 26 | 27 | def is_windows() -> bool: 28 | """Guess if windows""" 29 | platform_string = platform.system() 30 | return os.name == "nt" or platform_string == "Windows" or "_NT" in platform_string 31 | 32 | 33 | def is_powershell() -> bool: 34 | """ 35 | Check if parent process or other ancestor process is powershell 36 | """ 37 | # ref https://stackoverflow.com/a/55598796/33264 38 | # Get the parent process name. 39 | 40 | try: 41 | process_name = psutil.Process(os.getppid()).name() 42 | grand_process_name = psutil.Process(os.getppid()).parent().name() 43 | # See if it is Windows PowerShell (powershell.exe) or PowerShell Core 44 | # (pwsh[.exe]): 45 | is_that_shell = bool(re.fullmatch("pwsh|pwsh.exe|powershell.exe", process_name)) 46 | if not is_that_shell: 47 | is_that_shell = bool( 48 | re.fullmatch("pwsh|pwsh.exe|powershell.exe", grand_process_name) 49 | ) 50 | except psutil.NoSuchProcess: 51 | inform("Can't tell if this is powershell, assuming not.") 52 | is_that_shell = False 53 | return is_that_shell 54 | 55 | 56 | def check_is_aws() -> bool: 57 | """ 58 | Look at domain name to see if this is an ec2 machine 59 | """ 60 | # HACK: environment variable checking is much, much faster & reliable. 61 | name = socket.getfqdn() 62 | return "ip-" in name and ".ec2.internal" in name 63 | -------------------------------------------------------------------------------- /whats_that_code/parsing_based.py: -------------------------------------------------------------------------------- 1 | """ 2 | This only works for python and certain languages where there are easy to find parsers. 3 | 4 | I figure these are likely to be slow & explode the number of libs installed. 5 | 6 | TODO: 7 | c parser - pip install pycparser 8 | java parser - pip install plp.Java (abandoned?) 9 | php parser = pip install convert2php (maybe?) 10 | javascript parser = pip install pyesprima 11 | html = pip install py_w3c (html can be pretty bad and still 'parse') 12 | """ 13 | 14 | import ast 15 | import json 16 | from typing import List 17 | 18 | import defusedxml.ElementTree 19 | 20 | 21 | def parses_as_python(code: str) -> bool: 22 | """Validate by ast parsing""" 23 | try: 24 | _ = ast.parse(code) 25 | return True 26 | except SyntaxError: 27 | return False 28 | 29 | 30 | def parses_as_json(code: str) -> bool: 31 | """Validate by json.loads""" 32 | # pylint: disable=broad-except 33 | # noinspection PyBroadException 34 | try: 35 | _ = json.loads(code) 36 | return True 37 | except Exception: 38 | return False 39 | 40 | 41 | def parses_as_xml(code: str) -> bool: 42 | """If it parses as xml, it probably is xml or someting like it""" 43 | if "<" not in code or ">" not in code: 44 | # I don't care if a text fragment is xml in some sense 45 | return False 46 | # pylint: disable=broad-except 47 | # noinspection PyBroadException 48 | try: 49 | _ = defusedxml.ElementTree.fromstring(code, forbid_dtd=True) 50 | return True 51 | except Exception: # as ex: 52 | # print(ex) 53 | return False 54 | 55 | 56 | def language_by_parsing(code: str) -> List[str]: 57 | """Guess by parsing""" 58 | if not code or not code.strip(): 59 | return [] 60 | guesses = [] 61 | if parses_as_python(code): 62 | guesses.append("python") 63 | if parses_as_json(code): 64 | guesses.append("json") 65 | if parses_as_xml(code): 66 | guesses.append("xml") 67 | return guesses 68 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_black.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modifies code. 3 | - Modified code will have to be checked in. 4 | 5 | If code needs to be modified and we are on a build server, maybe break build. 6 | """ 7 | 8 | import shlex 9 | import sys 10 | from typing import Dict 11 | 12 | from navio_tasks.cli_commands import ( 13 | check_command_exists, 14 | config_pythonpath, 15 | execute, 16 | execute_get_text, 17 | ) 18 | from navio_tasks.settings import IS_GITLAB, PROJECT_NAME, VENV_SHELL 19 | from navio_tasks.system_info import is_windows, is_git_repo 20 | from navio_tasks.utils import inform 21 | 22 | 23 | def do_formatting(check: str, state: Dict[str, bool]) -> None: 24 | """ 25 | Format with black - this will not modify code if check is --check 26 | """ 27 | 28 | # check & format should be merged & use an arg 29 | # global FORMATTING_CHECK_DONE 30 | if state["check_already_done"]: 31 | inform("Formatting check says black will not reformat, so no need to repeat") 32 | return 33 | if sys.version_info < (3, 6): 34 | inform("Black doesn't work on python 2") 35 | return 36 | check_command_exists("black") 37 | 38 | command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace( 39 | " ", " " 40 | ) 41 | inform(command_text) 42 | command = shlex.split(command_text) 43 | if check: 44 | _ = execute(*command) 45 | state["check_already_done"] = True 46 | return 47 | result = execute_get_text(command, env=config_pythonpath()) 48 | assert result 49 | changed = [] 50 | for line in result.split("\n"): 51 | if "reformatted " in line: 52 | file = line[len("reformatted ") :].strip() 53 | changed.append(file) 54 | if not IS_GITLAB: 55 | if not is_git_repo("."): 56 | # don't need to git add anything because this isn't a git repo 57 | return 58 | for change in changed: 59 | if is_windows(): 60 | change = change.replace("\\", "/") 61 | command_text = f"git add {change}" 62 | inform(command_text) 63 | command = shlex.split(command_text) 64 | execute(*command) 65 | -------------------------------------------------------------------------------- /test/test_fast/test_language_guessing.py: -------------------------------------------------------------------------------- 1 | from whats_that_code.election import guess_language_all_methods 2 | from whats_that_code.keyword_based import guess_by_keywords 3 | from whats_that_code.guess_by_code_and_tags import assign_extension 4 | from whats_that_code.pygments_based import language_by_pygments 5 | from whats_that_code.regex_based import language_by_regex_features 6 | 7 | 8 | def test_assign_extension(): 9 | extension, language = assign_extension("print('hello')", tags=["python", "stuff"]) 10 | assert extension == ".py" 11 | 12 | 13 | def test_assign_extension2(): 14 | extension, language = assign_extension("", tags=["python", "stuff"]) 15 | assert extension == "" 16 | 17 | 18 | def test_assign_extension3(): 19 | # bash is hard to guess. 20 | extension, language = assign_extension("pip install foo", tags=["python", "stuff"]) 21 | assert extension == ".py" 22 | 23 | # extension,language= assign_extension("public static void class Foo {System.out.println('yo')}",True) 24 | # assert extension==".java" 25 | 26 | 27 | def test_election_bash(): 28 | # why did this stop working? 29 | assert guess_language_all_methods("sudo yum install pip", file_name="foo.sh") 30 | 31 | 32 | def test_election(): 33 | assert guess_language_all_methods( 34 | "public static void main(args[]){\n system.out.writeln('');}" 35 | ) 36 | 37 | 38 | def test_election2(): 39 | assert guess_language_all_methods("def yo():\n print('hello')", file_name="yo.py") 40 | 41 | 42 | # not handling file names with spaces in them?! 43 | # def test_election3(): 44 | # assert guess_language_all_methods( 45 | # "sudo yum install pip", surrounding_text="The file is named foo.py" 46 | # ) 47 | 48 | 49 | def test_guess_by_keywords(): 50 | assert "python" in guess_by_keywords("def class pip") 51 | 52 | 53 | def test_language_by_regex_features(): 54 | assert "python" in language_by_regex_features("def foo():\n print('yo')") 55 | 56 | 57 | def test_language_by_pygments(): 58 | assert language_by_pygments( 59 | """// Your First C++ Program 60 | 61 | #include 62 | 63 | int main() { 64 | std::cout << "Hello World!"; 65 | return 0; 66 | }""" 67 | ) 68 | -------------------------------------------------------------------------------- /navio_tasks/mutating_commands/cli_precommit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Precommit is a generic checker/build tool. It can contain a few or lots of 3 | tools, many of them not python specific. 4 | """ 5 | 6 | import shlex 7 | import sys 8 | 9 | from navio_tasks.cli_commands import ( 10 | check_command_exists, 11 | config_pythonpath, 12 | execute, 13 | execute_get_text, 14 | ) 15 | from navio_tasks.output import say_and_exit 16 | from navio_tasks.settings import VENV_SHELL 17 | from navio_tasks.system_info import is_git_repo 18 | from navio_tasks.utils import inform 19 | 20 | 21 | def do_precommit(is_interactive: bool) -> None: 22 | """ 23 | Build time execution of pre-commit checks. Modifies code so run before linter. 24 | """ 25 | if not is_interactive: 26 | inform("Not running precommit because it changes files") 27 | return 28 | check_command_exists("pre-commit") 29 | 30 | if is_git_repo("."): 31 | # don't try to install because it isn't a git repo 32 | command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ") 33 | inform(command_text) 34 | command = shlex.split(command_text) 35 | execute(*command) 36 | 37 | command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ") 38 | inform(command_text) 39 | command = shlex.split(command_text) 40 | result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) 41 | assert result 42 | changed = [] 43 | for line in result.split("\n"): 44 | if "changed " in line: 45 | file = line[len("reformatted ") :].strip() 46 | changed.append(file) 47 | if "FAILED" in result: 48 | inform(result) 49 | say_and_exit("Pre-commit Failed", "pre-commit") 50 | sys.exit(-1) 51 | 52 | if is_interactive: 53 | if not is_git_repo("."): 54 | # don't need to git add anything because this isn't a git repo 55 | return 56 | for change in changed: 57 | command_text = f"git add {change}" 58 | inform(command_text) 59 | # this breaks on windows! 60 | # command = shlex.split(command_text) 61 | execute(*command_text.split()) 62 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/cli_scspell.py: -------------------------------------------------------------------------------- 1 | """ 2 | Spell check. Most work for something that doesn't change application behavior. 3 | """ 4 | 5 | import os 6 | import shlex 7 | 8 | from navio_tasks import settings as settings 9 | from navio_tasks.cli_commands import config_pythonpath, execute_get_text 10 | from navio_tasks.settings import PROBLEMS_FOLDER, PROJECT_NAME, VENV_SHELL 11 | from navio_tasks.utils import inform 12 | 13 | 14 | def do_spell_check() -> None: 15 | """ 16 | Check spelling using scspell (pip install scspell3k) 17 | """ 18 | # tool can't recurse through files 19 | # tool returns a hard to parse format 20 | # tool has a really cumbersome way of adding values to dictionary 21 | walk_dir = PROJECT_NAME 22 | files_to_check = [] 23 | inform(walk_dir) 24 | for root, _, files in os.walk(walk_dir): 25 | if "pycache" in root: 26 | continue 27 | for file in files: 28 | inform(root + "/" + file) 29 | if file.endswith(".py"): 30 | files_to_check.append(root + "/" + file) 31 | 32 | files_to_check_string = " ".join(files_to_check) 33 | command_text = ( 34 | f"{VENV_SHELL} scspell --report-only " 35 | "--override-dictionary=spelling_dictionary.txt " 36 | f"--use-builtin-base-dict {files_to_check_string}".strip().replace(" ", " ") 37 | ) 38 | inform(command_text) 39 | command = shlex.split(command_text) 40 | result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) 41 | with open(f"{PROBLEMS_FOLDER}/spelling.txt", "w+") as outfile: 42 | outfile.write( 43 | "\n".join( 44 | [ 45 | row 46 | for row in result.replace("\r", "").split("\n") 47 | if "dictionary" in row 48 | ] 49 | ) 50 | ) 51 | 52 | def read_file() -> None: 53 | with open(f"{settings.CONFIG_FOLDER}/spelling_dictionary.txt") as reading_file: 54 | reading_result = reading_file.read() 55 | inform( 56 | "\n".join( 57 | [row for row in reading_result.split("\n") if "dictionary" in row] 58 | ) 59 | ) 60 | 61 | read_file() 62 | -------------------------------------------------------------------------------- /navio_tasks/non_breaking_commands/cli_sonar.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sonar complains about 3 | - yaml formatting 4 | - code complexity 5 | - commented code 6 | - (dead code?) 7 | - duplicate code 8 | 9 | Not all of those are equally actionable 10 | """ 11 | 12 | import json 13 | import os 14 | import sys 15 | 16 | import requests 17 | 18 | from navio_tasks.cli_commands import execute 19 | from navio_tasks.output import say_and_exit 20 | from navio_tasks.settings import PROJECT_NAME, VENV_SHELL 21 | from navio_tasks.system_info import is_windows 22 | from navio_tasks.utils import inform 23 | 24 | 25 | def do_sonar() -> str: 26 | """ 27 | Upload code to sonar for review 28 | """ 29 | sonar_key = os.environ["SONAR_KEY"] 30 | if is_windows(): 31 | command_name = "sonar-scanner.bat" 32 | else: 33 | command_name = "sonar-scanner" 34 | command = ( 35 | f"{VENV_SHELL} {command_name} " 36 | f"-Dsonar.login={sonar_key} " 37 | "-Dproject.settings=" 38 | "sonar-project.properties".strip().replace(" ", " ").split(" ") 39 | ) 40 | inform(command) 41 | execute(*command) 42 | url = ( 43 | "https://code-quality-test.loc.gov/api/issues/search?" 44 | f"componentKeys=public_record_{PROJECT_NAME}&resolved=false" 45 | ) 46 | 47 | session = requests.Session() 48 | session.auth = (sonar_key, "") 49 | 50 | response = session.get(url) 51 | 52 | errors_file = "sonar.json" 53 | with open(errors_file, "w+") as file_handle: 54 | inform(response.text) 55 | text = response.text 56 | if not text: 57 | say_and_exit("Failed to check for sonar", "sonar") 58 | sys.exit(-1) 59 | file_handle.write(text) 60 | 61 | try: 62 | with open(errors_file) as file_handle: 63 | data = json.load(file_handle) 64 | 65 | if data["issues"]: 66 | for result in data["issues"]: 67 | inform( 68 | "{} : {} line {}-{}".format( 69 | result["component"], 70 | result["message"], 71 | result["textRange"]["startLine"], 72 | result["textRange"]["endLine"], 73 | ) 74 | ) 75 | say_and_exit("sonar has issues with this code", "sonar") 76 | except json.JSONDecodeError: 77 | pass 78 | return "Sonar done" 79 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | on: [ push ] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v3 11 | - uses: actions/setup-python@v4 12 | with: 13 | python-version: '3.11' 14 | cache: 'pipenv' # caching pipenv dependencies 15 | - name: Install pipenv and pipx 16 | run: | 17 | pip install pipenv && pip install pipx 18 | 19 | - name: Install global dependencies 20 | run: | 21 | pipx install isort && pipx install black && pipx install bandit && pipx install pylint && \ 22 | pipx install pre-commit && pipx install pygount && pipx install vulture && \ 23 | pipx install flake8 && \ 24 | pipx inject flake8 dlint mccabe pyflakes pep8-naming flake8-bugbear && \ 25 | pipx install "pipenv-to-requirements==0.9.*" && \ 26 | pipx inject pipenv-to-requirements "pipenv==2022.9.8" && \ 27 | pipx install safety && pipx install pyupgrade && pipx install poetry 28 | 29 | - name: Install Dependencies 30 | run: pipenv install --dev --skip-lock 31 | 32 | - name: Run nb 33 | run: pipenv run nb package 34 | - name: Upload Package 35 | uses: actions/upload-artifact@v3.1.2 36 | with: 37 | name: packages 38 | path: dist/ 39 | if-no-files-found: error 40 | retention-days: 1 41 | 42 | 43 | pypi-publish: 44 | name: Upload release to PyPI 45 | runs-on: ubuntu-latest 46 | environment: 47 | name: pypi 48 | url: https://pypi.org/p/whats_that_code 49 | permissions: 50 | id-token: write # IMPORTANT: this permission is mandatory for trusted publishing 51 | # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 52 | steps: 53 | - name: Get packages 54 | uses: actions/download-artifact@v3.0.2 55 | with: 56 | name: packages 57 | path: dist/ 58 | - name: Publish package distributions to PyPI 59 | uses: pypa/gh-action-pypi-publish@release/v1 60 | needs: build 61 | -------------------------------------------------------------------------------- /navio_tasks/pure_reports/cli_pygount.py: -------------------------------------------------------------------------------- 1 | """ 2 | Lines fo code counting 3 | """ 4 | 5 | import os 6 | import shlex 7 | import subprocess 8 | 9 | from navio_tasks import settings as settings 10 | from navio_tasks.cli_commands import check_command_exists, prepinform_simple 11 | from navio_tasks.output import say_and_exit 12 | from navio_tasks.settings import REPORTS_FOLDER 13 | from navio_tasks.utils import inform 14 | 15 | 16 | def total_loc() -> int: 17 | """ 18 | Get Lines of Code for app 19 | """ 20 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 21 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 22 | # pylint: disable=bare-except 23 | try: 24 | with open( 25 | f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt" 26 | ) as file_handle: 27 | total_loc_value = file_handle.read() 28 | except: # noqa: B001 29 | do_count_lines_of_code() 30 | with open( 31 | f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt" 32 | ) as file_handle: 33 | total_loc_value = file_handle.read() 34 | 35 | return int(total_loc_value) 36 | 37 | 38 | def do_count_lines_of_code() -> None: 39 | """ 40 | Scale failure cut offs based on Lines of Code 41 | """ 42 | command_name = "pygount" 43 | check_command_exists(command_name) 44 | command_text = prepinform_simple(command_name) 45 | 46 | # keep out of src tree, causes extraneous change detections 47 | if not os.path.exists(f"{REPORTS_FOLDER}"): 48 | os.makedirs(f"{REPORTS_FOLDER}") 49 | output_file_name = f"{REPORTS_FOLDER}/line_counts.txt" 50 | command = shlex.split(command_text) 51 | with open(output_file_name, "w") as outfile: 52 | subprocess.call(command, stdout=outfile) 53 | 54 | with open(output_file_name) as file_handle: 55 | lines = sum(int(line.split("\t")[0]) for line in file_handle if line != "\n") 56 | 57 | total_loc_local = lines 58 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 59 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 60 | with open( 61 | f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt", "w+" 62 | ) as state_file: 63 | state_file.write(str(total_loc_local)) 64 | 65 | inform(f"Lines of code: {total_loc_local}") 66 | if total_loc_local == 0: 67 | say_and_exit( 68 | "No code found to build or package. Maybe the PROJECT_NAME is wrong?", 69 | "lines of code", 70 | ) 71 | -------------------------------------------------------------------------------- /.config/.license_rules: -------------------------------------------------------------------------------- 1 | [Licenses] 2 | # lower case 3 | authorized_licenses : 4 | bsd 5 | new bsd 6 | bsd license 7 | new bsd license 8 | simplified bsd 9 | apache 10 | apache 2.0 11 | apache software license 12 | dual 13 | gnu lgpl 14 | lgpl with exceptions or zpl 15 | isc license 16 | isc license (iscl) 17 | mit 18 | mit license 19 | python software foundation license 20 | zpl 2.1 21 | gpl v3 22 | apache license version 2.0 23 | apache license 2.0 24 | apache license, version 2.0 25 | lgpl 26 | mpl-2.0 27 | apache software 28 | mozilla public license 2.0 (mpl 2.0) 29 | gnu library or lesser general public license (lgpl) 30 | bsd-3-clause 31 | bsd-like 32 | Public Domain 33 | GNU Lesser General Public License v3 (LGPLv3) 34 | LGPL v3 35 | MIT 36 | license.txt 37 | public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt) 38 | GNU General Public License (GPL) 39 | BSD 40 | Python Software Foundation 41 | Apache Software 42 | BSD or Apache License, Version 2.0 43 | BSD 44 | BSD-2-Clause or Apache-2.0 45 | Python 46 | Python Software Foundation 47 | Public Domain 48 | ASL 2 49 | Apache Software 50 | Historical Permission Notice and Disclaimer (HPND) 51 | HPND 52 | Mozilla Public License 1.1 (MPL 1.1) 53 | GNU General Public License v2 or later (GPLv2+) 54 | GNU Lesser General Public License v2 or later (LGPLv2+) 55 | Python Software Foundation 56 | PSFL 57 | GNU Lesser General Public License v3 or later (LGPLv3+) 58 | LGPLv3+ 59 | LGPLv3 60 | GNU Lesser General Public License v3 (LGPLv3) 61 | GPLv2 62 | GNU General Public License v2 (GPLv2) 63 | BSD 3-Clause 64 | GNU GPL 3 65 | GNU General Public License (GPL) 66 | 3-Clause BSD 67 | BSD-2-Clause 68 | ISC 69 | GNU General Public License v2 or later (GPLv2+) 70 | Artistic 71 | GNU General Public License (GPL) 72 | MIT -or- Apache License 2.0 73 | Apache Software 74 | Apache-2.0 75 | Apache Software License 2.0 76 | LGPL+BSD 77 | BSD 78 | GNU Library or Lesser General Public License (LGPL) 79 | MIT/Apache-2.0 80 | http://www.apache.org/licenses/LICENSE-2.0 81 | Python Software Foundation 82 | PSF 83 | MPL 2.0 84 | Mozilla Public License 2.0 (MPL 2.0) 85 | 86 | unauthorized_licenses : 87 | foobar 88 | 89 | [Authorized Packages] 90 | # Python software license (see http://zesty.ca/python/uuid.README.txt) 91 | uuid : 1.30 92 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_detect_secrets.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | """ 8 | 9 | import json 10 | import shlex 11 | import sys 12 | 13 | from navio_tasks.cli_commands import ( 14 | check_command_exists, 15 | config_pythonpath, 16 | execute_get_text, 17 | ) 18 | from navio_tasks.output import say_and_exit 19 | from navio_tasks.settings import PROBLEMS_FOLDER, VENV_SHELL 20 | from navio_tasks.utils import inform 21 | 22 | 23 | def do_detect_secrets() -> str: 24 | """ 25 | Call detect-secrets tool 26 | 27 | I think this is the problem: 28 | 29 | # Code expects to stream output to file and then expects 30 | # interactive person, so code hangs. But also hangs in git-bash 31 | detect-secrets scan test_data/config.env > foo.txt 32 | detect-secrets audit foo.txt 33 | """ 34 | inform("Detect secrets broken ... can't figure out why") 35 | return "nope" 36 | 37 | # pylint: disable=unreachable 38 | check_command_exists("detect-secrets") 39 | errors_file = f"{PROBLEMS_FOLDER}/detect-secrets-results.txt" 40 | command_text = ( 41 | f"{VENV_SHELL} detect-secrets scan " 42 | "--base64-limit 4 " 43 | # f"--exclude-files .idea|.min.js|.html|.xsd|" 44 | # f"lock.json|.scss|Pipfile.lock|.secrets.baseline|" 45 | # f"{PROBLEMS_FOLDER}/lint.txt|{errors_file}".strip().replace(" ", " ") 46 | ) 47 | inform(command_text) 48 | command = shlex.split(command_text) 49 | 50 | with open(errors_file, "w") as outfile: 51 | env = config_pythonpath() 52 | output = execute_get_text(command, ignore_error=False, env=env) 53 | outfile.write(output) 54 | # subprocess.call(command, stdout=outfile, env=env) 55 | 56 | with open(errors_file, "w+") as file_handle: 57 | text = file_handle.read() 58 | if not text: 59 | say_and_exit("Failed to check for secrets", "detect-secrets") 60 | sys.exit(-1) 61 | file_handle.write(text) 62 | 63 | try: 64 | with open(errors_file) as json_file: 65 | data = json.load(json_file) 66 | 67 | if data["results"]: 68 | for result in data["results"]: 69 | inform(result) 70 | say_and_exit( 71 | "detect-secrets has discovered high entropy strings, " 72 | "possibly passwords?", 73 | "detect-secrets", 74 | ) 75 | except json.JSONDecodeError: 76 | pass 77 | return "Detect secrets completed." 78 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_get_secrets.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | ------- 8 | """ 9 | 10 | import shlex 11 | import subprocess 12 | 13 | from navio_tasks.cli_commands import check_command_exists, execute, is_cmd_exe 14 | from navio_tasks.settings import IS_GITLAB 15 | from navio_tasks.system_info import check_is_aws, is_git_repo 16 | from navio_tasks.utils import inform 17 | 18 | 19 | def do_git_secrets() -> str: 20 | """ 21 | Install git secrets if possible. 22 | """ 23 | if is_cmd_exe(): 24 | inform("git secrets is a bash script, only works in bash (or maybe PS") 25 | return "skipped git secrets, this is cmd.exe shell" 26 | # not sure how to check for a git subcommand 27 | if not is_git_repo("."): 28 | inform("This is not a git repo, won't run git-secrets") 29 | return "Not a git repo, skipped" 30 | check_command_exists("git") 31 | 32 | if check_is_aws(): 33 | # no easy way to install git secrets on ubuntu. 34 | return "This is AWS, not doing git-secrets" 35 | if IS_GITLAB: 36 | inform("Nothing is edited on gitlab build server") 37 | return "This is gitlab, not doing git-secrets" 38 | try: 39 | # check to see if secrets even is a git command 40 | 41 | commands = ["git secrets --install", "git secrets --register-aws"] 42 | for command in commands: 43 | command_parts = shlex.split(command) 44 | command_process = subprocess.run( 45 | command_parts, 46 | capture_output=True, 47 | check=True, 48 | ) 49 | for stream in [command_process.stdout, command_process.stderr]: 50 | if stream: 51 | for line in stream.decode().split("\n"): 52 | inform("*" + line) 53 | except subprocess.CalledProcessError as cpe: 54 | inform(cpe) 55 | installed = False 56 | for stream in [cpe.stdout, cpe.stderr]: 57 | if stream: 58 | for line in stream.decode().split("\n"): 59 | inform("-" + line) 60 | if "commit-msg already exists" in line: 61 | inform("git secrets installed.") 62 | installed = True 63 | break 64 | if not installed: 65 | raise 66 | command_text = "git secrets --scan -r ./".strip().replace(" ", " ") 67 | command_parts = shlex.split(command_text) 68 | execute(*command_parts) 69 | return "git-secrets succeeded" 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | error* 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | .build_state 107 | lint.txt 108 | mypy_errors.txt 109 | 110 | # this churns too much 111 | coverage 112 | 113 | # also churns too much 114 | detect-secrets-results.txt 115 | dead_code.txt 116 | dist 117 | dist/ 118 | .idea 119 | .idea/ 120 | 121 | # sonar stuff 122 | .scannerwork 123 | line_counts.txt 124 | 125 | pip-wheel-metadata 126 | .cache 127 | missing_key.txt 128 | missing.txt 129 | sonar.json 130 | .vscode 131 | data/reprocess_files 132 | data/differential_files 133 | *.sqlite3 134 | 135 | .DS_Store 136 | **/.DS_Store 137 | sonar-unit-test-results.xml 138 | 139 | .DS_Store/ 140 | .DS_Store 141 | data/ 142 | !/test/data/sample_files/ 143 | package/ 144 | output 145 | 146 | problems. 147 | reports/ 148 | examples 149 | 150 | .config/reqs.txt 151 | 152 | .config/requirements_for_safety.txt 153 | 154 | .config/requirements-dev-tox.txt 155 | 156 | .config/requirements-dev.txt 157 | 158 | .config/requirements.txt 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # whats_that_code 2 | This is a programming language detection library. 3 | 4 | It will detect programming language of source in pure python from an ensemble of classifiers. 5 | Use this when a quick and dirty first approximation is good enough. 6 | whats_that_code can currently identify 60%+ of samples without knowing the extension or tag. 7 | 8 | I created this because I wanted 9 | - a pure python programming language detector 10 | - no machine learning dependencies 11 | 12 | Tested on python 3.6 through 3.9. 13 | 14 | Badges 15 | ------ 16 | [![Libraries.io SourceRank](https://img.shields.io/librariesio/sourcerank/pypi/whats_that_code?longCache=true&style=flat-square)](https://libraries.io/github/matthewdeanmartin/whats_that_code/sourcerank) 17 | 18 | [![Downloads](https://static.pepy.tech/badge/whats-that-code/month)](https://pepy.tech/project/whats-that-code) 19 | 20 | 21 | [![CodeFactor](https://www.codefactor.io/repository/github/matthewdeanmartin/whats_that_code/badge)](https://www.codefactor.io/repository/github/matthewdeanmartin/whats_that_code) 22 | 23 | ## Usage 24 | ```python 25 | from whats_that_code.election import guess_language_all_methods 26 | code = "def yo():\n print('hello')" 27 | result = guess_language_all_methods(code, file_name="yo.py") 28 | assert result == ["python"] 29 | ``` 30 | 31 | ## How it Works 32 | 1) Inspects file extension if available. 33 | 2) Inspects shebang 34 | 3) Looks for keywords 35 | 4) Counts regexes for common patterns 36 | 5) Attemps to parse python, json, yaml 37 | 6) Inspects tags if available. 38 | 39 | Each is imperfect and can error. The classifier then combines the results of each using a voting algorithm 40 | 41 | This works best if you only use it for fallback, e.g. classifying code that can't already be classified by extension or tag, 42 | or when tag is ambiguous. 43 | 44 | It was a tool that outgrew being a part of [so_pip](https://github.com/matthewdeanmartin/so_pip) a StackOverflow code 45 | extraction tool I wrote. 46 | 47 | ## Docs 48 | - [TODO](https://github.com/matthewdeanmartin/whats_that_code/tree/main/docs/TODO.md) 49 | - [LICENSE](https://github.com/matthewdeanmartin/whats_that_code/tree/main/LICENSE) 50 | - [Prior Art](https://github.com/matthewdeanmartin/whats_that_code/tree/main/docs/prior_art.md) Every similar project/tool, including defunct 51 | - [ChangeLog](https://github.com/matthewdeanmartin/whats_that_code/tree/main/docs/CHANGES.md) 52 | 53 | ## Notable Similar Tools 54 | - [Guesslang](https://pypi.org/project/guesslang/) - python and tensorflow driven solution. Reasonable results but 55 | slow startup and not pure python. 56 | - [pygments](https://pygments.org/docs/api/#pygments.lexers.guess_lexer) pure python, but sometimes lousy identification 57 | rates. 58 | -------------------------------------------------------------------------------- /test/test_slow/test_classify_all.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classify 'em 3 | """ 4 | 5 | import os 6 | 7 | from whats_that_code.election import guess_language_all_methods 8 | from whats_that_code.known_languages import FILE_EXTENSIONS 9 | import pytest 10 | 11 | 12 | def locate_file(file_name: str, executing_file: str) -> str: 13 | """ 14 | Find file relative to a source file, e.g. 15 | locate("foo/bar.txt", __file__) 16 | 17 | Succeeds regardless to context of execution 18 | 19 | File must exist 20 | """ 21 | file_path = os.path.join( 22 | os.path.dirname(os.path.abspath(executing_file)), file_name 23 | ) 24 | if not os.path.exists(file_path): 25 | raise TypeError(file_path + " doesn't exist") 26 | return file_path 27 | 28 | 29 | def test_with_source_classifier(): 30 | total = 0 31 | right = 0 32 | wrong = 0 33 | try: 34 | files = locate_file("../../examples/sourceclassifier-master/sources", __file__) 35 | except TypeError: 36 | pytest.skip("don't have a bunch of files downloaded to verify against") 37 | 38 | if not os.path.exists(files): 39 | pytest.skip("don't have a bunch of files downloaded to verify against") 40 | 41 | for subdir, dirs, files in os.walk(files): 42 | 43 | for file_name in files: 44 | # if not "java" in file_name: 45 | # continue 46 | file_path = subdir + os.sep + file_name 47 | with open(file_path, encoding="utf-8", errors="ignore") as code_file: 48 | total += 1 49 | guess = guess_language_all_methods(code_file.read()) 50 | neither = True 51 | if guess in ["c", "cpp"] and ( 52 | file_name.endswith(".gcc") or file_name.endswith(".gcc") 53 | ): 54 | right += 1 55 | neither = False 56 | elif guess in ["php", "xml", "html"] and ( 57 | file_name.endswith(".php") 58 | or file_name.endswith(".xml") 59 | or file_name.endswith(".html") 60 | ): 61 | right += 1 62 | neither = False 63 | elif guess in FILE_EXTENSIONS: 64 | extensions = FILE_EXTENSIONS[guess] 65 | one_does = False 66 | for extension in extensions: 67 | if file_name.endswith(extension): 68 | one_does = True 69 | if not one_does: 70 | print(file_path, guess) 71 | wrong += 1 72 | neither = False 73 | 74 | else: 75 | right += 1 76 | neither = False 77 | 78 | # 68% !! 79 | print(f"right {right}... {right/total}%") 80 | assert right / total > 0.50 81 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3 3 | { 4 | "name": "Python 3", 5 | "forwardPorts": [ 6 | 8000 7 | ], 8 | "build": { 9 | "dockerfile": "Dockerfile", 10 | "context": ".." 11 | }, 12 | // Configure tool-specific properties. 13 | "customizations": { 14 | // Configure properties specific to VS Code. 15 | "vscode": { 16 | // Set *default* container specific settings.json values on container create. 17 | "settings": { 18 | "python.defaultInterpreterPath": "/usr/local/bin/python", 19 | "python.linting.enabled": true, 20 | "python.linting.pylintEnabled": true, 21 | "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", 22 | "python.formatting.blackPath": "/usr/local/py-utils/bin/black", 23 | "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", 24 | "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", 25 | "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", 26 | "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", 27 | "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", 28 | "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", 29 | "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", 30 | "python.testing.unittestEnabled": false, 31 | "python.testing.pytestEnabled": true, 32 | "workbench.startupEditor": "newUntitledFile" 33 | }, 34 | // Add the IDs of extensions you want installed when the container is created. 35 | "extensions": [ 36 | "ms-python.python", 37 | "ms-python.vscode-pylance" 38 | ] 39 | } 40 | }, 41 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 42 | // "forwardPorts": [], 43 | 44 | // Use 'postCreateCommand' to run commands after the container is created. 45 | "postCreateCommand": "pipenv sync --dev", 46 | // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. 47 | "remoteUser": "vscode", 48 | // "folders": [ 49 | // { 50 | // "path": ".devcontainer" 51 | // } 52 | // ], 53 | "features": { 54 | "ghcr.io/devcontainers-contrib/features/black:2": {}, 55 | "ghcr.io/devcontainers-contrib/features/coverage-py:2": {}, 56 | "ghcr.io/devcontainers-contrib/features/isort:2": {}, 57 | "ghcr.io/devcontainers-contrib/features/pipenv:2": {}, 58 | "ghcr.io/devcontainers-contrib/features/pylint:2": {}, 59 | "ghcr.io/devcontainers/features/sshd:1": { 60 | "version": "latest" 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_mypy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Type checking. Hardest to fix, most likely to find a real bug. 3 | """ 4 | 5 | import subprocess 6 | import sys 7 | from typing import List 8 | 9 | from navio_tasks.cli_commands import check_command_exists 10 | from navio_tasks.output import say_and_exit 11 | from navio_tasks.pure_reports.cli_pygount import total_loc 12 | 13 | # Anything that feels like governance/policy should be in build.py 14 | from navio_tasks.settings import PROBLEMS_FOLDER, PROJECT_NAME, VENV_SHELL 15 | from navio_tasks.utils import inform 16 | 17 | 18 | def do_mypy() -> str: 19 | """ 20 | Are types ok? 21 | """ 22 | check_command_exists("mypy") 23 | if sys.version_info < (3, 4): 24 | inform("Mypy doesn't work on python < 3.4") 25 | return "command is missing" 26 | command = ( 27 | f"{VENV_SHELL} mypy {PROJECT_NAME} " 28 | "--ignore-missing-imports " 29 | "--strict".strip().replace(" ", " ") 30 | ) 31 | inform(command) 32 | bash_process = subprocess.Popen( 33 | command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE 34 | ) 35 | out, _ = bash_process.communicate() # wait 36 | mypy_file = f"{PROBLEMS_FOLDER}/all_mypy_errors.txt" 37 | with open(mypy_file, "w", encoding="utf-8") as out_file: 38 | out_file.write(out.decode()) 39 | return mypy_file 40 | 41 | 42 | def evaluated_mypy_results( 43 | mypy_file: str, small_code_base_cutoff: int, maximum_mypy: int, skips: List[str] 44 | ) -> str: 45 | """ 46 | Decided if the mypy is bad enough to stop the build. 47 | """ 48 | with open(mypy_file) as out_file: 49 | out = out_file.read() 50 | 51 | def contains_a_skip(line_value: str) -> bool: 52 | """ 53 | Should this line be skipped 54 | """ 55 | # skips is a closure 56 | for skip in skips: 57 | if skip in line_value or line_value.startswith(skip): 58 | return True 59 | return False 60 | 61 | actually_bad_lines: List[str] = [] 62 | total_lines = 0 63 | with open(mypy_file, "w+") as lint_file: 64 | lines = out.split("\n") 65 | for line in lines: 66 | total_lines += 1 67 | if contains_a_skip(line): 68 | continue 69 | if not line.startswith(PROJECT_NAME): 70 | continue 71 | actually_bad_lines.append(line) 72 | lint_file.writelines([line]) 73 | 74 | num_lines = len(actually_bad_lines) 75 | if total_loc() > small_code_base_cutoff: 76 | max_lines = maximum_mypy 77 | else: 78 | max_lines = 2 # off by 1 right now 79 | 80 | if num_lines > max_lines: 81 | for line in actually_bad_lines: 82 | inform(line) 83 | say_and_exit(f"Too many lines of mypy : {num_lines}, max {max_lines}", "mypy") 84 | sys.exit(-1) 85 | 86 | if num_lines == 0 and total_lines == 0: 87 | # should always have at least 'found 0 errors' in output 88 | say_and_exit( 89 | "No mypy warnings at all, did mypy fail to run or is it installed?", "mypy" 90 | ) 91 | sys.exit(-1) 92 | return "mypy succeeded" 93 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '29 1 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Use only 'java' to analyze code written in Java, Kotlin or both 38 | # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both 39 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v3 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v2 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | 54 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 55 | # queries: security-extended,security-and-quality 56 | 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@v2 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 65 | 66 | # If the Autobuild fails above, remove it and uncomment the following three lines. 67 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 68 | 69 | # - run: | 70 | # echo "Run, Build Application using script" 71 | # ./location_of_script_within_repo/buildscript.sh 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@v2 75 | with: 76 | category: "/language:${{matrix.language}}" 77 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_pytest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Only reports 3 | No file changes 4 | Should break build on any issues. 5 | Expect few issues 6 | All issues should be addressable immediately. 7 | ---------- 8 | """ 9 | 10 | import multiprocessing 11 | 12 | from navio_tasks.cli_commands import ( 13 | check_command_exists, 14 | config_pythonpath, 15 | execute_with_environment, 16 | ) 17 | from navio_tasks.settings import ( 18 | IS_GITLAB, 19 | IS_INTERNAL_NETWORK, 20 | MINIMUM_TEST_COVERAGE, 21 | PROJECT_NAME, 22 | REPORTS_FOLDER, 23 | VENV_SHELL, 24 | RUN_ALL_TESTS_REGARDLESS_TO_NETWORK, 25 | ) 26 | from navio_tasks.utils import inform 27 | 28 | 29 | def do_pytest() -> None: 30 | """ 31 | Pytest and coverage, which replaces nose tests 32 | """ 33 | check_command_exists("pytest") 34 | 35 | # Somedays VPN just isn't there. 36 | if IS_INTERNAL_NETWORK or RUN_ALL_TESTS_REGARDLESS_TO_NETWORK: 37 | fast_only = False 38 | else: 39 | fast_only = True 40 | if fast_only: 41 | test_folder = "test/test_fast" 42 | minimum_coverage = 48 43 | else: 44 | test_folder = "test" 45 | minimum_coverage = MINIMUM_TEST_COVERAGE 46 | 47 | my_env = config_pythonpath() 48 | 49 | command = ( 50 | f"{VENV_SHELL} pytest {test_folder} -v " 51 | f"--junitxml={REPORTS_FOLDER}/sonar-unit-test-results.xml " 52 | "--cov-report xml " 53 | f"--cov={PROJECT_NAME} " 54 | f"--cov-fail-under {minimum_coverage}".strip().replace(" ", " ") 55 | + " --quiet" # 15000 pages of call stack don't help anyone 56 | ) 57 | # when it works, it is FAST. when it doesn't, we get lots of timeouts. 58 | # if not IS_GITLAB: 59 | # command += f" -n {multiprocessing.cpu_count()} " 60 | if not IS_GITLAB: 61 | command += " -n 2 " 62 | 63 | inform(command) 64 | execute_with_environment(command, my_env) 65 | inform("Tests will not be re-run until code changes. Run pynt reset to force.") 66 | 67 | 68 | def do_pytest_coverage(fast_only: bool) -> None: 69 | """ 70 | Just the coverage report 71 | """ 72 | # this is failing on windows 73 | # check_command_exists("py.test") 74 | 75 | my_env = config_pythonpath() 76 | 77 | # generate report separate from cov-fail-under step. 78 | # py.test incorrectly reports 0.00 coverage, 79 | # but only when reports are generated. 80 | 81 | # Coverage report is (sometimes) broken. Need alternative to py.test? 82 | # This is consuming too much time to figure out why it 83 | # collects no tests & then fails on code 5 84 | 85 | if fast_only: 86 | test_folder = "test/test_fast" 87 | else: 88 | test_folder = "test" 89 | 90 | command = ( 91 | f"{VENV_SHELL} pytest {test_folder} -v " 92 | "--cov-report html:coverage " 93 | f"--cov={PROJECT_NAME}".strip().replace(" ", " ") 94 | ) 95 | if not IS_GITLAB: 96 | command += f" -n {multiprocessing.cpu_count()} " 97 | inform(command) 98 | execute_with_environment(command, my_env) 99 | inform("Coverage will not rerun until code changes. Run `pynt reset` to force") 100 | -------------------------------------------------------------------------------- /navio_tasks/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Settings loaded from pynt.ini 3 | 4 | Maybe move to pyproject.toml? 5 | """ 6 | 7 | import configparser 8 | import os 9 | import platform 10 | 11 | from navio_tasks.network import check_public_ip, is_known_network 12 | 13 | CONFIG_FOLDER = ".config" 14 | 15 | 16 | def load_config() -> configparser.SectionProxy: 17 | """ 18 | Load config 19 | """ 20 | config = configparser.ConfigParser() 21 | config.read(f"{CONFIG_FOLDER}/.pynt") 22 | return config["DEFAULT"] 23 | 24 | 25 | SECTION = load_config() 26 | PROJECT_NAME = SECTION["PROJECT_NAME"] 27 | 28 | SRC = SECTION["SRC"] 29 | PROBLEMS_FOLDER = SECTION["PROBLEMS_FOLDER"] 30 | REPORTS_FOLDER = SECTION["REPORTS_FOLDER"] 31 | IS_SHELL_SCRIPT_LIKE = SECTION["IS_SHELL_SCRIPT_LIKE"].lower() in ["true", "1"] 32 | COMPLEXITY_CUT_OFF = SECTION["COMPLEXITY_CUT_OFF"] 33 | 34 | MINIMUM_TEST_COVERAGE = int(SECTION["MINIMUM_TEST_COVERAGE"]) 35 | SMALL_CODE_BASE_CUTOFF = int(SECTION["SMALL_CODE_BASE_CUTOFF"]) 36 | MAXIMUM_LINT = int(SECTION["MAXIMUM_LINT"]) 37 | MAXIMUM_MYPY = int(SECTION["MAXIMUM_MYPY"]) 38 | MAXIMUM_DEAD_CODE = int(SECTION["MAXIMUM_DEAD_CODE"]) 39 | MAXIMUM_MANIFEST_ERRORS = int(SECTION["MAXIMUM_MANIFEST_ERRORS"]) 40 | 41 | VENV_SHELL = SECTION["VENV_SHELL"] 42 | 43 | 44 | PACKAGE_WITH = SECTION["PACKAGE_WITH"] 45 | if PACKAGE_WITH not in ["poetry", "setup.py", "None", "none"]: 46 | raise TypeError("PACKAGE_WITH must be poetry, setup.py or None") 47 | if PACKAGE_WITH in ["None", "none"]: 48 | PACKAGE_WITH = "" 49 | 50 | # pylint: disable=simplifiable-if-expression 51 | # uh... need mechanism to show preference for no venv, poetry or pipenv. 52 | # WANT_TO_USE_PIPENV = True if VENV_SHELL else False 53 | PIPENV_ACTIVE = "PIPENV_ACTIVE" in os.environ and os.environ["PIPENV_ACTIVE"] == "1" 54 | POETRY_ACTIVE = "POETRY_ACTIVE" in os.environ and os.environ["POETRY_ACTIVE"] == "1" 55 | PIP_ACTIVE = "VIRTUAL_ENV" in os.environ and not PIPENV_ACTIVE and not POETRY_ACTIVE 56 | TOX_ACTIVE = "TOX_PACKAGE" in os.environ 57 | if PIPENV_ACTIVE: 58 | INSTALL_WITH = "pipenv" 59 | elif POETRY_ACTIVE: 60 | INSTALL_WITH = "poetry" 61 | elif PIP_ACTIVE or TOX_ACTIVE: 62 | INSTALL_WITH = "pip" 63 | else: 64 | for key, value in os.environ.items(): 65 | print(key, value) 66 | raise TypeError( 67 | "neither pipenv, poetry, nor pip virtual env active. " 68 | "Do we really want to use the system python?" 69 | ) 70 | 71 | 72 | if PIPENV_ACTIVE or POETRY_ACTIVE: 73 | # activating each run is very, very slow. 74 | VENV_SHELL = "" 75 | 76 | PYTHON = "python" 77 | IS_DJANGO = False 78 | IS_GITLAB = "GITLAB_CI" in os.environ or "CI" in os.environ 79 | IS_WINDOWS = platform.system() == "Windows" 80 | IS_ALPINE_DOCKER = os.path.exists("/etc/alpine-release") 81 | IS_JENKINS = "FROM_JENKINS" in os.environ and os.environ["FROM_JENKINS"] == "TRUE" 82 | 83 | CURRENT_HASH = None 84 | 85 | VENDOR_LIBS = ":" 86 | 87 | # so that formatting doesn't run after check done once 88 | FORMATTING_CHECK_DONE = False 89 | 90 | # network call 91 | PUBLIC_IP = check_public_ip() 92 | KNOWN_IP_PREFIX = SECTION["KNOWN_IP_PREFIX"] 93 | RUN_ALL_TESTS_REGARDLESS_TO_NETWORK = ( 94 | SECTION["RUN_ALL_TESTS_REGARDLESS_TO_NETWORK"] == "True" 95 | ) 96 | IS_INTERNAL_NETWORK = is_known_network(KNOWN_IP_PREFIX) or PUBLIC_IP.startswith( 97 | KNOWN_IP_PREFIX 98 | ) 99 | IS_INTERACTIVE = not (IS_GITLAB or IS_JENKINS) 100 | 101 | SPEAK_WHEN_BUILD_FAILS = SECTION["SPEAK_WHEN_BUILD_FAILS"] == "True" 102 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "whats_that_code" 3 | version = "0.2.0" 4 | description = "Guess programming language from a string or file." 5 | authors = ["Matthew Martin "] 6 | keywords = ["language detection","programming language detection"] 7 | classifiers = [ 8 | "Development Status :: 4 - Beta", 9 | "Intended Audience :: Developers", 10 | "License :: OSI Approved :: MIT License", 11 | "Operating System :: OS Independent", 12 | "Programming Language :: Python", 13 | "Programming Language :: Python :: 3", 14 | "Programming Language :: Python :: 3.6", 15 | "Programming Language :: Python :: 3.7", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Python :: 3.11", 20 | "Programming Language :: Python :: 3.12", 21 | ] 22 | include = [ 23 | "whats_that_code/**/*.py", 24 | "whats_that_code/**/*.md", 25 | "whats_that_code/**/*.txt", 26 | "whats_that_code/**/*.html", 27 | "whats_that_code/**/*.jinja", 28 | ] 29 | exclude = [ 30 | 31 | ] 32 | license = "MIT" 33 | readme = "README.md" 34 | repository = "https://github.com/matthewdeanmartin/whats_that_code" 35 | homepage = "https://github.com/matthewdeanmartin/whats_that_code" 36 | documentation ="https://github.com/matthewdeanmartin/whats_that_code" 37 | 38 | [tool.poetry.urls] 39 | "Bug Tracker" = "https://github.com/matthewdeanmartin/whats_that_code/issues" 40 | "Change Log" = "https://github.com/matthewdeanmartin/whats_that_code/blob/main/docs/CHANGES.md" 41 | 42 | [tool.poetry.scripts] 43 | #guess_that_code = 'whats_that_code.__main__:main' 44 | 45 | [tool.poetry.dependencies] 46 | python = "^3.6" 47 | pyrankvote = "*" 48 | defusedxml = "*" 49 | pygments = "*" 50 | 51 | [tool.poetry.dev-dependencies] 52 | pytest = "==6.0.1" 53 | pytest-cov = ">=2.10.1" 54 | pytest-timeout = "*" 55 | pytest-xdist = ">=2.1.0" 56 | pip-check = "==2.6" 57 | checksumdir = "==1.1.7" 58 | requests = "==2.22.0" 59 | dodgy = "==0.2.1" 60 | gitchangelog = "==3.0.4" 61 | liccheck = "==0.4.3" 62 | psutil = "==5.6.7" 63 | pebble = "==4.5.0" 64 | python-dotenv = "==0.11.0" 65 | gitpython = "*" 66 | ifaddr = "*" 67 | vendorize = "*" 68 | 69 | [tool.black] 70 | line-length = 88 71 | target-version = ['py37'] 72 | include = '\.pyi?$' 73 | exclude = ''' 74 | 75 | ( 76 | /( 77 | \.eggs # exclude a few common directories in the 78 | | \.git # root of the project 79 | | \.hg 80 | | \.mypy_cache 81 | | \.tox 82 | | \.venv 83 | | _build 84 | | buck-out 85 | | build 86 | | dist 87 | )/ 88 | | foo.py # also separately exclude a file named foo.py in 89 | # the root of the project 90 | ) 91 | ''' 92 | [build-system] 93 | requires = ["poetry>=0.12"] 94 | build-backend = "poetry.masonry.api" 95 | 96 | [tool.pytest.ini_options] 97 | minversion = "6.0" 98 | testpaths = [ 99 | "tests", 100 | ] 101 | junit_family = "xunit1" 102 | norecursedirs = ["vendor", "scripts"] 103 | # don't know how to do this in toml 104 | #addopts = "--strict-markers" 105 | #markers = 106 | # slow: marks tests as slow (deselect with '-m "not slow"') 107 | # fast: marks tests as fast (deselect with '-m "not fast"') 108 | 109 | [tool.isort] 110 | default_section = "THIRDPARTY" 111 | force_grid_wrap = 0 112 | include_trailing_comma = true 113 | known_first_party = ["so_pip"] 114 | line_length = 88 115 | multi_line_output = 3 116 | use_parentheses = true 117 | -------------------------------------------------------------------------------- /navio_tasks/commands/cli_pylint.py: -------------------------------------------------------------------------------- 1 | """ 2 | Lots of code gripes. 3 | """ 4 | 5 | import os 6 | import shlex 7 | import subprocess 8 | import sys 9 | from typing import List 10 | 11 | from navio_tasks import settings as settings 12 | from navio_tasks.cli_commands import check_command_exists, config_pythonpath 13 | from navio_tasks.output import say_and_exit 14 | from navio_tasks.pure_reports.cli_pygount import total_loc 15 | from navio_tasks.settings import ( 16 | IS_DJANGO, 17 | IS_GITLAB, 18 | PROBLEMS_FOLDER, 19 | PROJECT_NAME, 20 | VENV_SHELL, 21 | ) 22 | from navio_tasks.utils import inform 23 | 24 | 25 | def do_lint(folder_type: str) -> str: 26 | """ 27 | Execute pylint 28 | """ 29 | # pylint: disable=too-many-locals 30 | check_command_exists("pylint") 31 | if folder_type == PROJECT_NAME: 32 | pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc" 33 | lint_output_file_name = f"{PROBLEMS_FOLDER}/lint.txt" 34 | else: 35 | pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc_{folder_type}" 36 | lint_output_file_name = f"{PROBLEMS_FOLDER}/lint_{folder_type}.txt" 37 | 38 | if os.path.isfile(lint_output_file_name): 39 | os.remove(lint_output_file_name) 40 | 41 | if IS_DJANGO: 42 | django_bits = "--load-plugins pylint_django " 43 | else: 44 | django_bits = "" 45 | 46 | # pylint: disable=pointless-string-statement 47 | command_text = ( 48 | f"{VENV_SHELL} pylint {django_bits} " f"--rcfile={pylintrc} {folder_type} " 49 | ) 50 | 51 | command_text += " " 52 | "--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" 53 | "".strip().replace(" ", " ") 54 | 55 | inform(command_text) 56 | command = shlex.split(command_text) 57 | 58 | with open(lint_output_file_name, "w") as outfile: 59 | env = config_pythonpath() 60 | subprocess.call(command, stdout=outfile, env=env) 61 | return lint_output_file_name 62 | 63 | 64 | def evaluated_lint_results( 65 | lint_output_file_name: str, 66 | small_code_base_cut_off: int, 67 | maximum_lint: int, 68 | fatals: List[str], 69 | ) -> str: 70 | """Deciding if the lint is bad enough to fail 71 | Also treats certain errors as fatal even if under the maximum cutoff. 72 | """ 73 | with open(lint_output_file_name) as file_handle: 74 | full_text = file_handle.read() 75 | lint_did_indeed_run = "Your code has been rated at" in full_text 76 | 77 | with open(lint_output_file_name) as file_handle: 78 | fatal_errors = sum(1 for line in file_handle if ": E" in line or ": F" in line) 79 | for fatal in fatals: 80 | for line in file_handle: 81 | if fatal in file_handle or ": E" in line or ": F" in line: 82 | fatal_errors += 1 83 | 84 | if fatal_errors > 0: 85 | with open(lint_output_file_name) as file_handle: 86 | for line in file_handle: 87 | if "*************" in line: 88 | continue 89 | if not line or not line.strip("\n "): 90 | continue 91 | inform(line.strip("\n ")) 92 | 93 | message = f"Fatal lint errors and possibly others, too : {fatal_errors}" 94 | if IS_GITLAB: 95 | with open(lint_output_file_name) as error_file: 96 | inform(error_file.read()) 97 | say_and_exit(message, "lint") 98 | return message 99 | with open(lint_output_file_name) as lint_file_handle: 100 | for line in [ 101 | line 102 | for line in lint_file_handle 103 | if not ( 104 | "*************" in line 105 | or "---------------------" in line 106 | or "Your code has been rated at" in line 107 | or line == "\n" 108 | ) 109 | ]: 110 | inform(line) 111 | 112 | if total_loc() > small_code_base_cut_off: 113 | cutoff = maximum_lint 114 | else: 115 | cutoff = 0 116 | with open(lint_output_file_name) as lint_file_handle: 117 | num_lines = sum( 118 | 1 119 | for line in lint_file_handle 120 | if not ( 121 | "*************" in line 122 | or "---------------------" in line 123 | or "Your code has been rated at" in line 124 | or line == "\n" 125 | ) 126 | ) 127 | if num_lines > cutoff: 128 | say_and_exit(f"Too many lines of lint : {num_lines}, max {cutoff}", "pylint") 129 | sys.exit(-1) 130 | with open(lint_output_file_name) as lint_file_handle: 131 | num_lines_all_output = sum(1 for _ in lint_file_handle) 132 | if ( 133 | not lint_did_indeed_run 134 | and num_lines_all_output == 0 135 | and os.path.isfile(lint_output_file_name) 136 | ): 137 | # should always have at least 'found 0 errors' in output 138 | 139 | # force lint to re-run, because empty file will be missing 140 | os.remove(lint_output_file_name) 141 | say_and_exit( 142 | "No lint messages at all, did pylint fail to run or is it installed?", 143 | "pylint", 144 | ) 145 | sys.exit(-1) 146 | 147 | return "pylint succeeded" 148 | -------------------------------------------------------------------------------- /whats_that_code/election.py: -------------------------------------------------------------------------------- 1 | """ 2 | Ranked choice election 3 | """ 4 | 5 | from typing import List, Optional 6 | 7 | import pyrankvote 8 | from pyrankvote import Ballot, Candidate 9 | 10 | from whats_that_code.extension_based import guess_by_extension 11 | from whats_that_code.guess_by_popularity import language_by_popularity 12 | from whats_that_code.keyword_based import guess_by_keywords 13 | from whats_that_code.known_languages import FILE_EXTENSIONS 14 | from whats_that_code.parsing_based import parses_as_xml 15 | from whats_that_code.pygments_based import language_by_pygments 16 | from whats_that_code.regex_based import language_by_regex_features 17 | from whats_that_code.shebang_based import language_by_shebang 18 | from whats_that_code.tag_based import match_tag_to_languages 19 | 20 | 21 | def guess_language_all_methods( 22 | code: str, 23 | file_name: str = "", 24 | surrounding_text: str = "", 25 | tags: Optional[List[str]] = None, 26 | priors: Optional[List[str]] = None, 27 | ) -> Optional[str]: 28 | """ 29 | Choose language with multiple algorithms via ranked choice. 30 | 31 | Ensemble classifier in fancy talk. 32 | """ 33 | 34 | # very smart voters 35 | vote_by_shebang = language_by_shebang(code) 36 | vote_by_extension = guess_by_extension(file_name=file_name) 37 | vote_by_extension_in_text = guess_by_extension(text=surrounding_text) 38 | vote_by_tags = match_tag_to_languages(tags) 39 | 40 | # mid-tier voters 41 | vote_by_priors = guess_by_prior_knowledge(priors) 42 | vote_by_regex_features = language_by_regex_features(code) 43 | 44 | all_but_stupid = set( 45 | vote_by_tags 46 | + vote_by_shebang 47 | + vote_by_extension 48 | + vote_by_extension_in_text 49 | + vote_by_regex_features 50 | + vote_by_priors 51 | ) 52 | # stupid voters 53 | vote_by_keyword = guess_by_keywords(code) 54 | # dumb voter block can't double their impact 55 | vote_by_pygments = [ 56 | _ for _ in language_by_pygments(code) if _ not in vote_by_keyword 57 | ] 58 | 59 | # Only want to hear from stupid voters if no one else votes 60 | if all_but_stupid: 61 | vote_by_keyword = [] 62 | vote_by_pygments = [] 63 | 64 | all_possible = set( 65 | vote_by_tags 66 | + vote_by_shebang 67 | + vote_by_extension 68 | + vote_by_extension_in_text 69 | + vote_by_regex_features 70 | + vote_by_priors 71 | # don't include stupid voters 72 | ) 73 | # above keeps wanting to guess the obscure languages. 74 | vote_by_popularity = language_by_popularity(all_possible) 75 | 76 | all_vote_lists = [ 77 | # if this has any info, in probably is really good. Give 'em two votes 78 | vote_by_tags, 79 | vote_by_shebang, 80 | vote_by_extension, 81 | vote_by_extension_in_text, 82 | # ad hoc way to give smart algo's more votes 83 | vote_by_tags, 84 | vote_by_shebang, 85 | vote_by_extension, 86 | vote_by_extension_in_text, 87 | # mid tier 88 | vote_by_priors, 89 | vote_by_regex_features, 90 | vote_by_popularity, 91 | # dummies 92 | vote_by_keyword, 93 | vote_by_pygments, 94 | ] 95 | 96 | # validate votes, get list of everything voted for 97 | for votes in all_vote_lists: 98 | for item in votes: 99 | all_possible.add(item) 100 | for vote in votes: 101 | if vote.lower() != vote: 102 | raise TypeError("Bad casing") 103 | if "." in vote: 104 | raise TypeError("Name not extension") 105 | 106 | # TODO: more comprehensive way to deal with "clones" 107 | # handle xml & php 108 | can_be_xml = True 109 | if "xml" in all_possible: 110 | if not parses_as_xml(code): 111 | for votes in all_vote_lists: 112 | if "xml" in votes: 113 | votes.remove("xml") 114 | can_be_xml = False 115 | 116 | # convert to ballots and hold election 117 | candidates = {} 118 | ballots = [] 119 | for ballot in all_vote_lists: 120 | for candidate in ballot: 121 | candidates[candidate] = Candidate(candidate) 122 | if ballot: 123 | ranked_ballot = Ballot(ranked_candidates=[candidates[_] for _ in ballot]) 124 | ballots.append(ranked_ballot) 125 | # else abstains 126 | 127 | if len(candidates) == 1: 128 | return list(candidates.values())[0].name 129 | 130 | election_result = pyrankvote.instant_runoff_voting( 131 | list(candidates.values()), ballots 132 | ) 133 | 134 | winners = election_result.get_winners() 135 | if not winners: 136 | return None 137 | 138 | if not can_be_xml and winners[0].name == "xml": 139 | raise TypeError() 140 | return winners[0].name 141 | 142 | 143 | def guess_by_prior_knowledge(priors: Optional[List[str]]) -> List[str]: 144 | """Let user tell us what he thinks are likely""" 145 | vote_by_priors = [] 146 | if priors: 147 | for prior in priors: 148 | if prior in FILE_EXTENSIONS: 149 | if prior not in vote_by_priors: 150 | vote_by_priors.append(prior) 151 | else: 152 | print(f"{prior} is not a known programming language") 153 | return vote_by_priors 154 | -------------------------------------------------------------------------------- /navio_tasks/cli_commands.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generic tools for handling shell commands, as opposed to commands that 3 | can be executed via `import tool` 4 | """ 5 | 6 | import io 7 | import os 8 | import re 9 | import shlex 10 | import shutil 11 | import subprocess 12 | import sys 13 | from contextlib import redirect_stderr, redirect_stdout 14 | from typing import Dict, Iterable, List, Optional, Tuple 15 | 16 | import psutil 17 | 18 | from navio_tasks import settings as settings 19 | from navio_tasks.settings import PROJECT_NAME, VENV_SHELL 20 | from navio_tasks.system_info import is_powershell 21 | from navio_tasks.utils import inform 22 | 23 | 24 | def config_pythonpath() -> Dict[str, str]: 25 | """ 26 | Add to PYTHONPATH 27 | """ 28 | my_env = { 29 | "PYTHONIOENCODING": "utf-8", 30 | "LC_ALL": "en_US.UTF-8", 31 | "LANG": "en_US.UTF-8", 32 | "PYTHONDONTWRITEBYTECODE": "", 33 | } 34 | for key, value in os.environ.items(): 35 | my_env[key] = value 36 | my_env["PYTHONPATH"] = my_env.get("PYTHONPATH", "") # + VENDOR_LIBS 37 | return my_env 38 | 39 | 40 | def is_cmd_exe() -> bool: 41 | """ 42 | Check if parent process or other ancestor process is cmd 43 | """ 44 | # ref https://stackoverflow.com/a/55598796/33264 45 | # Get the parent process name. 46 | try: 47 | process_name = psutil.Process(os.getppid()).name() 48 | grand_process_name = psutil.Process(os.getppid()).parent().name() 49 | # pylint: disable=bare-except, broad-except 50 | try: 51 | great_grand_process_name = ( 52 | psutil.Process(os.getppid()).parent().parent().name() 53 | ) 54 | except: # noqa: B001 55 | great_grand_process_name = "No great grandparent" 56 | 57 | inform(process_name, grand_process_name, great_grand_process_name) 58 | is_that_shell = bool(re.fullmatch("cmd|cmd.exe", process_name)) 59 | if not is_that_shell: 60 | is_that_shell = bool(re.fullmatch("cmd|cmd.exe", grand_process_name)) 61 | if not is_that_shell: 62 | is_that_shell = bool(re.fullmatch("cmd|cmd.exe", great_grand_process_name)) 63 | except psutil.NoSuchProcess: 64 | inform("Can't tell if this is cmd.exe, assuming not.") 65 | is_that_shell = False 66 | return is_that_shell 67 | 68 | 69 | def check_command_exists( 70 | command: str, throw_on_missing: bool = False, exit_on_missing: bool = True 71 | ) -> bool: 72 | """ 73 | Check if exists by a variety of methods that vary by shell 74 | 75 | # can this be replaced with shutil.which? 76 | """ 77 | 78 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 79 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 80 | state_file = f"{settings.CONFIG_FOLDER}/.build_state/exists_" + command + ".txt" 81 | if os.path.exists(state_file): 82 | return True 83 | 84 | # when I originally wrote this function, I didn't know about shutil.which 85 | # if it performs well across many OSs, then we can phase out the rest. 86 | if shutil.which(command): 87 | already_found(command) 88 | return True 89 | venv_shell = [_ for _ in VENV_SHELL.split(" ") if _ != ""] 90 | 91 | # Build command & print. Must do here or the print gets redirected! 92 | # these commands lack a --version. 93 | if command in ["pyroma", "liccheck", "pipenv_to_requirements", "pyupgrade", "pyt"]: 94 | # will fail unless bash shell 95 | if is_powershell(): 96 | cmd = venv_shell + ["powershell", "get-command", command] 97 | else: 98 | cmd = venv_shell + ["which", command] 99 | else: 100 | cmd = venv_shell + [command, "--version"] 101 | 102 | # pylint: disable=broad-except 103 | try: 104 | with io.StringIO() as buf, io.StringIO() as buf2, redirect_stdout( 105 | buf 106 | ), redirect_stderr(buf2): 107 | # execute command built up above. 108 | _ = subprocess.check_output(cmd) 109 | output = buf.getvalue() 110 | output2 = buf2.getvalue() 111 | inform(output, output2) 112 | if "not recognized" in output or "not recognized" in output2: 113 | inform(f"Got error checking if {command} exists") 114 | if throw_on_missing: 115 | raise TypeError("Can't find command") 116 | if exit_on_missing: 117 | sys.exit(-1) 118 | return False 119 | already_found(command) 120 | except OSError as os_error: 121 | inform(os_error) 122 | inform(f"Got error checking if {command} exists") 123 | if throw_on_missing: 124 | raise 125 | if exit_on_missing: 126 | sys.exit(-1) 127 | return False 128 | # pylint: disable=broad-except 129 | except Exception as ex: 130 | inform("Other error") 131 | inform(ex) 132 | inform(f"Got error checking if {command} exists") 133 | if throw_on_missing: 134 | raise 135 | if exit_on_missing: 136 | sys.exit(-1) 137 | return False 138 | return True 139 | 140 | 141 | def already_found(command: str) -> None: 142 | """Check if we already checked if this command exists.""" 143 | with open( 144 | f"{settings.CONFIG_FOLDER}/.build_state/exists_" + command + ".txt", "w+" 145 | ) as handle: 146 | handle.write("OK") 147 | 148 | 149 | def prepinform_simple(command: str, no_project: bool = False) -> str: 150 | """ 151 | Deal with simple command that only takes project name as arg 152 | """ 153 | if no_project: 154 | command = f"{VENV_SHELL} {command}".strip().replace(" ", " ") 155 | else: 156 | command = f"{VENV_SHELL} {command} {PROJECT_NAME}".strip().replace(" ", " ") 157 | inform(command) 158 | return command 159 | 160 | 161 | def execute_get_text( 162 | command: List[str], 163 | ignore_error: bool = False, 164 | # shell: bool = True, # causes cross plat probs, security warnings, etc. 165 | env: Optional[Dict[str, str]] = None, 166 | ) -> str: 167 | """ 168 | Execute shell command and return stdout txt 169 | """ 170 | if env is None: 171 | env = {} 172 | 173 | completed = None 174 | try: 175 | completed = subprocess.run( 176 | command, 177 | check=not ignore_error, 178 | capture_output=True, 179 | env=env, 180 | ) 181 | except subprocess.CalledProcessError: 182 | if ignore_error and completed: 183 | return completed.stdout.decode("utf-8") + completed.stderr.decode("utf-8") 184 | raise 185 | else: 186 | return completed.stdout.decode("utf-8") + completed.stderr.decode("utf-8") 187 | 188 | 189 | # from pynt.contrib 190 | def execute(script: str, *args: Iterable[str]) -> int: 191 | """ 192 | Executes a command through the shell. Spaces should breakup the args. 193 | Usage: execute('grep', 'TODO', '*') 194 | """ 195 | 196 | popen_args = [script] + list(args) 197 | # pylint: disable=broad-except 198 | try: 199 | return subprocess.check_call(popen_args, shell=False) # noqa 200 | except subprocess.CalledProcessError as ex: 201 | inform(ex) 202 | sys.exit(ex.returncode) 203 | except Exception as ex: 204 | inform(f"Error: {ex} with script: {script} and args {args}") 205 | sys.exit(1) 206 | 207 | 208 | def execute_with_environment(command: str, env: Dict[str, str]) -> Tuple[bytes, bytes]: 209 | """ 210 | Yet another helper to execute a command 211 | """ 212 | # Python 2 code! Python 3 uses context managers. 213 | command_text = command.strip().replace(" ", " ") 214 | command_parts = shlex.split(command_text) 215 | shell_process = subprocess.Popen(command_parts, env=env) 216 | value = shell_process.communicate() # wait 217 | if shell_process.returncode != 0: 218 | inform(f"Didn't get a zero return code, got : {shell_process.returncode}") 219 | sys.exit(-1) 220 | return value 221 | -------------------------------------------------------------------------------- /navio_tasks/build_state.py: -------------------------------------------------------------------------------- 1 | """ 2 | Make best efforts to decide if a task needs to run again, or if it is pointless 3 | because the source code hasn't changed. 4 | 5 | Sometimes task depends on the world, not the source code 6 | 7 | Strategies 8 | - folder or source tree changed 9 | - file changed 10 | - list of files changed 11 | 12 | Is it worse is lint counting. Move to different module? 13 | """ 14 | 15 | import functools 16 | import hashlib 17 | import os 18 | import shutil 19 | import sys 20 | import time 21 | from typing import Any, Callable, Optional, TypeVar, cast 22 | 23 | from checksumdir import dirhash 24 | 25 | from navio_tasks import settings as settings 26 | from navio_tasks.settings import PROJECT_NAME, SRC 27 | from navio_tasks.utils import inform 28 | 29 | # pylint: disable=invalid-name 30 | FuncType = Callable[..., Any] 31 | # pylint: disable=invalid-name 32 | F = TypeVar("F", bound=FuncType) 33 | CURRENT_HASH = None 34 | 35 | # bash to find what has change recently 36 | # find src/ -type f -print0 | xargs -0 stat -f "%m %N" | sort -rn | head -10 | 37 | # cut -f2- -d" " 38 | 39 | 40 | class BuildState: 41 | """ 42 | Try not to re-do what doesn't need to be redone 43 | """ 44 | 45 | def __init__(self, what: str, directory: str) -> None: 46 | """ 47 | Set initial state 48 | """ 49 | self.what = what 50 | self.directory = directory 51 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 52 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 53 | self.state_file_name = ( 54 | f"{settings.CONFIG_FOLDER}/.build_state/last_change_{what}.txt" 55 | ) 56 | 57 | def oh_never_mind(self) -> None: 58 | """ 59 | If a task fails, we don't care if it didn't change since last, re-run, 60 | """ 61 | # noinspection PyBroadException 62 | # pylint: disable=bare-except 63 | try: 64 | os.remove(self.state_file_name) 65 | except: # noqa: B001 66 | pass 67 | 68 | def has_source_code_tree_changed(self) -> bool: 69 | """ 70 | If a task succeeds & is re-run and didn't change, we might not 71 | want to re-run it if it depends *only* on source code 72 | """ 73 | # pylint: disable=global-statement 74 | global CURRENT_HASH 75 | directory = self.directory 76 | 77 | # if CURRENT_HASH is None: 78 | # inform("hashing " + directory) 79 | # inform(os.listdir(directory)) 80 | CURRENT_HASH = dirhash( 81 | directory, 82 | "md5", 83 | ignore_hidden=True, 84 | # changing these exclusions can cause dirhas to skip EVERYTHING 85 | # excluded_files=[".coverage", "lint.txt"], 86 | excluded_extensions=[".pyc"], 87 | ) 88 | 89 | inform("Searching " + self.state_file_name) 90 | if os.path.isfile(self.state_file_name): 91 | with open(self.state_file_name, "r+") as file: 92 | last_hash = file.read() 93 | if last_hash != CURRENT_HASH: 94 | file.seek(0) 95 | file.write(CURRENT_HASH) 96 | file.truncate() 97 | return True 98 | return False 99 | 100 | # no previous file, by definition not the same. 101 | with open(self.state_file_name, "w") as file: 102 | file.write(CURRENT_HASH) 103 | return True 104 | 105 | 106 | def oh_never_mind(what: str) -> None: 107 | """ 108 | If task fails, remove file that says it was recently run. 109 | Needs to be like this because tasks can change code (and change the hash) 110 | """ 111 | state = BuildState(what, PROJECT_NAME) 112 | state.oh_never_mind() 113 | 114 | 115 | def has_source_code_tree_changed( 116 | task_name: str, expect_file: Optional[str] = None 117 | ) -> bool: 118 | """ 119 | Hash source code tree to know if it has changed 120 | 121 | Also check if an expected output file exists or not. 122 | """ 123 | if expect_file: 124 | if os.path.isdir(expect_file) and not os.listdir(expect_file): 125 | os.path.dirname(expect_file) 126 | # output folder empty 127 | return True 128 | if not os.path.isfile(expect_file): 129 | # output file gone 130 | return True 131 | state = BuildState(task_name, os.path.join(SRC, PROJECT_NAME)) 132 | return state.has_source_code_tree_changed() 133 | 134 | 135 | def skip_if_no_change(name: str, expect_files: Optional[str] = None) -> F: 136 | """ 137 | Don't run decorated task if nothing in the source has changed. 138 | """ 139 | 140 | # https://stackoverflow.com/questions/5929107/decorators-with-parameters 141 | def real_decorator(func: F) -> F: 142 | """Wrapper""" 143 | 144 | @functools.wraps(func) 145 | def wrapper(*args: Any, **kwargs: Any) -> Callable: 146 | """Wrapper""" 147 | if not has_source_code_tree_changed(name, expect_files): 148 | inform("Nothing changed, won't re-" + name) 149 | return lambda x: None 150 | try: 151 | return func(*args, **kwargs) 152 | except: # noqa: B001 153 | oh_never_mind(name) 154 | raise 155 | 156 | return cast(F, wrapper) 157 | 158 | return cast(F, real_decorator) 159 | 160 | 161 | def hash_it(path: str) -> str: 162 | """ 163 | Hash a single file. Return constant if it doesn't exist. 164 | """ 165 | if not os.path.exists(path): 166 | return "DOESNOTEXIST" 167 | with open(path, "rb") as file_handle: 168 | return hashlib.sha256(file_handle.read()).hexdigest() 169 | 170 | 171 | def skip_if_this_file_does_not_change(name: str, file: str) -> F: 172 | """ 173 | Skip decorated task if this referenced file didn't change. Useful 174 | if a task depends on a single file and not (potentially) any file in the source tree 175 | """ 176 | 177 | def real_decorator(func: F) -> F: 178 | """Wrapper""" 179 | 180 | @functools.wraps(func) 181 | def wrapper(*args: Any, **kwargs: Any) -> Callable: 182 | """Wrapper""" 183 | state_file = ( 184 | f"{settings.CONFIG_FOLDER}/.build_state/file_hash_" + name + ".txt" 185 | ) 186 | previous_hash = "catdog" 187 | if os.path.exists(state_file): 188 | with open(state_file) as old_file: 189 | previous_hash = old_file.read() 190 | 191 | new_hash = hash_it(file) 192 | if new_hash == previous_hash: 193 | inform("Nothing changed, won't re-" + name) 194 | return lambda x: f"Skipping {name}, no change" 195 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 196 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 197 | with open(state_file, "w+") as state: 198 | state.write(new_hash) 199 | try: 200 | 201 | return func(*args, **kwargs) 202 | except: # noqa: B001 203 | # reset if step fails 204 | os.remove(state_file) 205 | raise 206 | 207 | return cast(F, wrapper) 208 | 209 | return cast(F, real_decorator) 210 | 211 | 212 | def reset_build_state() -> None: 213 | """ 214 | Delete all .build_state & to force all steps to re-run next build 215 | """ 216 | if os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 217 | shutil.rmtree(f"{settings.CONFIG_FOLDER}/.build_state") 218 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 219 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 220 | 221 | 222 | def timed() -> F: 223 | """This decorator prints the execution time for the decorated function.""" 224 | 225 | def real_decorator(func: F) -> F: 226 | """Wrapper""" 227 | 228 | @functools.wraps(func) 229 | def wrapper(*args: Any, **kwargs: Any) -> Callable: 230 | """Wrapper""" 231 | start = time.time() 232 | result = func(*args, **kwargs) 233 | end = time.time() 234 | inform(f"{func.__name__} ran in {round(end - start, 2)}s") 235 | return result 236 | 237 | return cast(F, wrapper) 238 | 239 | return cast(F, real_decorator) 240 | 241 | 242 | def is_it_worse(task_name: str, current_rows: int, margin: int) -> bool: 243 | """ 244 | Logic for dealing with a code base with very large amounts of lint. 245 | 246 | You will never fix it all, just don't make it worse. 247 | """ 248 | if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): 249 | os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") 250 | file_name = f"{settings.CONFIG_FOLDER}/.build_state/last_count_{task_name}.txt" 251 | 252 | last_rows = sys.maxsize 253 | if os.path.isfile(file_name): 254 | with open(file_name, "r+") as file: 255 | last_rows = int(file.read()) 256 | if last_rows != current_rows: 257 | file.seek(0) 258 | file.write(str(current_rows)) 259 | file.truncate() 260 | 261 | return current_rows > (last_rows + margin) 262 | -------------------------------------------------------------------------------- /whats_that_code/codex_markers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Source: 3 | https://github.com/TomCrypto/Codex 4 | The MIT License (MIT) 5 | 6 | Copyright © 2016 Thomas BENETEAU 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 13 | """ 14 | 15 | import re 16 | from typing import Dict, List, Pattern 17 | 18 | 19 | def _compiled_regex(pattern: str, dotall: bool = True) -> Pattern[str]: 20 | """precompile some regex""" 21 | flags = (re.MULTILINE | re.DOTALL) if dotall else re.MULTILINE 22 | return re.compile(pattern, flags) 23 | 24 | 25 | MARKERS: Dict[str, List[Pattern[str]]] = { 26 | # Markers applicable to several languages 27 | # Code, not, say, English 28 | "code": [ 29 | _compiled_regex(r"^\s{2,}\S"), # indentation 30 | # TODO: need to explain what is going on here as it's not obvious 31 | _compiled_regex(r".{,1}\s*[=/\-\+\|<\{\}\[\](\)~`_\^#]+\s*.{,1}"), 32 | # generic symbol capture 33 | _compiled_regex(r"&&|\|\||>>|<<"), 34 | _compiled_regex(r"\+=|-=|/=|\*=|==|!="), 35 | _compiled_regex(r"__\w+"), 36 | _compiled_regex(r"{$"), 37 | _compiled_regex(r"^\s*};?"), 38 | _compiled_regex(r"if\s*\((.*?)\)\s*{"), 39 | _compiled_regex(r"for\s*\((.*?)\)\s*{"), 40 | _compiled_regex(r"0x\d+"), 41 | _compiled_regex(r"=\s*0x\d+"), 42 | ], 43 | # C preprocessor markers 44 | "c": [ 45 | _compiled_regex(r'^\s*#\s*include\s+("|<)[^">]+("|>)$'), 46 | _compiled_regex(r"^\s*#\s*include\s+<[^\.>]+>$"), 47 | # <> only without .h variant for C++ 48 | _compiled_regex(r"^\s*#\s*ifn?def\s+\w+$"), 49 | _compiled_regex(r"^\s*#\s*if\s+(.*?)$"), 50 | _compiled_regex(r"^\s*#\s*if\s+defined\((.*?)$"), 51 | _compiled_regex(r"^\s*#\s*define \w+(.*?)$"), 52 | _compiled_regex(r"^\s*#\s*endif$"), 53 | _compiled_regex(r"^\s*#\s*undef\s+\w+$"), 54 | _compiled_regex(r"^\s*#\s*else$"), 55 | _compiled_regex(r"^\s*#\s*pragma(.*?)$"), 56 | # C markers 57 | # TODO 58 | _compiled_regex(r"/\*(.*?)\*/"), 59 | ], 60 | "delphi": [ 61 | # Delphi markers 62 | # TODO: Delphi preprocessor markers 63 | _compiled_regex(r"{\$(.*?)}$"), 64 | _compiled_regex(r"^unit\s+\w;$"), 65 | _compiled_regex(r"^interface(\s+^uses(.*?))?;$"), 66 | _compiled_regex(r"\w+\s*=\s*(.*?);"), 67 | _compiled_regex(r"^\s*\w+\s*=\s*class\(\w+\)$"), 68 | _compiled_regex(r"^\s*\w+\s*=\s*class\(\w+\)$(.*?)^\s*end;$"), 69 | _compiled_regex( 70 | r"\s*\w+:\s*(Integer|integer|String|string|Boolean|boolean|Byte|byte|ShortInt|shortint|Word|word|SmallInt|smallint|LongWord|longword|Cardinal|cardinal|LongInt|longint|Int64|int64|Single|single|Double|double|Currency|currency|Extended|extended|Char|char|WideChar|widechar|AnsiChar|ansichar|ShortString|shortstring|AnsiString|ansistring|WideString|widestring|T\w+)(;|\))" 71 | ), 72 | _compiled_regex( 73 | r"(override|virtual|Override|Virtual|Overload|overload|Cdecl|cdecl|Stdcall|stdcall);" 74 | ), 75 | _compiled_regex(r"^\s*function\s*\w+(\((.*?)\))?\s*:\s*\w+;"), 76 | _compiled_regex(r"^\s*procedure\s*\w+(\((.*?)\))?;"), 77 | _compiled_regex(r"^\s*property\s+\w+\s*:\s*\w+(.*?);"), 78 | _compiled_regex(r"^\s*constructor Create;"), 79 | _compiled_regex(r"^\s*destructor Destroy;"), 80 | _compiled_regex(r"^\s*var(.*?)^\s*begin"), 81 | _compiled_regex(r"inherited(\s+\w+(\((.*?)\))?)?;"), 82 | _compiled_regex(r"^\s*begin(.*?)^\s*end"), 83 | _compiled_regex(r"\w+\s*:=\s*(.*?);"), 84 | _compiled_regex(r"\s<>\s"), 85 | _compiled_regex(r"\(\*(.*?)\*\)"), 86 | ], 87 | "python": [ 88 | # Python markers 89 | _compiled_regex( 90 | r"^(\s*from\s+[\.\w]+)?\s*import\s+[\*\.,\w]+(,\s*[\*\.,\w]+)*(\s+as\s+\w+)?$" 91 | ), 92 | _compiled_regex(r"^\s*def\s+\w+\((.*?):$", dotall=False), 93 | _compiled_regex(r"^\s*if\s(.*?):$(.*?)(^\s*else:)?$", dotall=False), 94 | _compiled_regex(r"^\s*if\s(.*?):$(.*?)(^\s*elif:)?$", dotall=False), 95 | _compiled_regex(r"^\s*try:$(.*?)^\s*except(.*?):"), 96 | _compiled_regex(r"True|False"), 97 | _compiled_regex(r"==\s*(True|False)"), 98 | _compiled_regex(r"is\s+(None|True|False)"), 99 | _compiled_regex(r"^\s*if\s+(.*?)\s+in[^:\n]+:$", dotall=False), 100 | _compiled_regex(r"^\s*pass$"), 101 | _compiled_regex(r"print\((.*?)\)$", dotall=False), 102 | _compiled_regex(r"^\s*for\s+\w+\s+in\s+(.*?):$"), 103 | _compiled_regex(r"^\s*class\s+\w+\s*(\([.\w]+\))?:$", dotall=False), 104 | _compiled_regex(r"^\s*@(staticmethod|classmethod|property)$"), 105 | _compiled_regex(r"__repr__"), 106 | _compiled_regex(r'"(.*?)"\s+%\s+(.*?)$', dotall=False), 107 | _compiled_regex(r"'(.*?)'\s+%\s+(.*?)$", dotall=False), 108 | _compiled_regex(r"^\s*raise\s+\w+Error(.*?)$"), 109 | _compiled_regex(r'"""(.*?)"""'), 110 | _compiled_regex(r"'''(.*?)'''"), 111 | _compiled_regex(r"\s*# (.*?)$"), 112 | _compiled_regex(r"^\s*import re$"), 113 | _compiled_regex(r"re\.\w+"), 114 | _compiled_regex(r"^\s*import time$"), 115 | _compiled_regex(r"time\.\w+"), 116 | _compiled_regex(r"^\s*import datetime$"), 117 | _compiled_regex(r"datetime\.\w+"), 118 | _compiled_regex(r"^\s*import random$"), 119 | _compiled_regex(r"random\.\w+"), 120 | _compiled_regex(r"^\s*import math$"), 121 | _compiled_regex(r"math\.\w+"), 122 | _compiled_regex(r"^\s*import os$"), 123 | _compiled_regex(r"os\.\w+"), 124 | _compiled_regex(r"^\s*import os.path$"), 125 | _compiled_regex(r"os\.path\.\w+"), 126 | _compiled_regex(r"^\s*import sys$"), 127 | _compiled_regex(r"sys\.\w+"), 128 | _compiled_regex(r"^\s*import argparse$"), 129 | _compiled_regex(r"argparse\.\w+"), 130 | _compiled_regex(r"^\s*import subprocess$"), 131 | _compiled_regex(r"subprocess\.\w+"), 132 | _compiled_regex(r'^\s*if\s+__name__\s*=\s*"__main__"\s*:$'), 133 | _compiled_regex(r"^\s*if\s+__name__\s*=\s*'__main__'\s*:$"), 134 | _compiled_regex(r"self\.\w+(\.\w+)*\((.*?)\)"), 135 | ], 136 | "haskell": [ 137 | # Haskell markers 138 | _compiled_regex(r"let\s+\w+\s*="), 139 | _compiled_regex(r"::\s+\w+\s+->"), 140 | _compiled_regex(r">>="), 141 | _compiled_regex(r"^\s*import(\s+qualified)?\s+[\.\w]+(\s*\((.*?))?$"), 142 | _compiled_regex(r"^\s*module\s+[\.\w]+(.*?)where$"), 143 | _compiled_regex(r"^\s*{-#(.*?)#-}"), 144 | _compiled_regex(r"^\s*\w+\s*::(.*?)$"), 145 | _compiled_regex(r"->\s+\[?[\w]+\]?"), 146 | _compiled_regex(r"\w+\s*<-\s*\w+"), 147 | _compiled_regex(r"\w+\s+\$\s+\w+"), 148 | _compiled_regex(r"\(\w+::\w+\)"), 149 | _compiled_regex(r"\w+\s+::\s+\w+"), 150 | _compiled_regex(r"\w+'"), 151 | _compiled_regex(r"<\$>"), 152 | _compiled_regex(r"^\s*=>\s+(.*?)$"), 153 | _compiled_regex(r"^\s*instance[^=>]+=>(.*?)where$"), 154 | _compiled_regex(r"^(.*?)=\s+do$", dotall=False), 155 | _compiled_regex(r"\+\+"), 156 | _compiled_regex(r"where$"), 157 | _compiled_regex(r"^\s*\|\s+\w+(.*?)=(.*?)$"), 158 | _compiled_regex(r"-- (.*?)$"), 159 | ], 160 | "xml": [ 161 | # XML markers 162 | _compiled_regex(r'<\w+\s*(\s+[:\.\-\w]+="[^"]*")*\s*>(.*?)<\s*/\w+\s*>'), 163 | _compiled_regex(r'<\s*/\w+\s*(\s+[:\.\-\w]+="[^"]*")*\s*>(.*?)<\s*/\w+\s*>'), 164 | _compiled_regex(r'<\w+\s*(\s+[:\.\-\w]+="[^"]*")*\s*/>'), 165 | _compiled_regex(r"<\?xml(.*?)\?>"), 166 | _compiled_regex(r""), 167 | ], 168 | "html": [ 169 | # HTML markers 170 | _compiled_regex(r""), 171 | _compiled_regex(r""), 172 | _compiled_regex(r"(.*?)"), 173 | _compiled_regex(r"(.*?)"), 174 | _compiled_regex(r"
(.*?)
"), 175 | _compiled_regex(r""), 176 | _compiled_regex(r"
"), 177 | _compiled_regex(r" "), 178 | _compiled_regex(r'(.*?)'), 179 | _compiled_regex(r'(.*?)'), 180 | _compiled_regex(r'(.*?)

'), 181 | _compiled_regex(r'(.*?)'), 182 | _compiled_regex(r'(.*?)'), 183 | _compiled_regex(r'(.*?)'), 184 | _compiled_regex(r'(.*?)'), 185 | _compiled_regex(r'(.*?)'), 186 | _compiled_regex(r'(.*?)'), 187 | _compiled_regex(r'(.*?)'), 188 | _compiled_regex(r'(.*?)'), 189 | _compiled_regex(r""), 190 | ], 191 | "json": [ 192 | # JSON markers 193 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*\[(.*?)\]'), 194 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*{(.*?)\}'), 195 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*[\.\-\deE]+\s*(,|}|\])'), 196 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*"[^"]*"\s*(,|}|\])'), 197 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*true\s*(,|}|\])'), 198 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*false\s*(,|}|\])'), 199 | _compiled_regex(r'(,|{|\[)?\s*"[^"]*"\s*:\s*null\s*(,|}|\])'), 200 | _compiled_regex(r"(({|\[)\s({|\[))+"), 201 | _compiled_regex(r"((}|\])\s(}|\]))+"), 202 | ], 203 | "javascript": [ 204 | # Javascript markers 205 | _compiled_regex(r"\w+\.get(.*?);"), 206 | _compiled_regex(r"\w+:\s*function\s*\((.*?)},?"), 207 | _compiled_regex(r"this\.\w+"), 208 | _compiled_regex(r"var\s+\w+(\s*,\s*\w+)*\s*=(.*?);$", dotall=False), 209 | _compiled_regex(r"[\.\w+]+\s*===\s*[\.\w+]+"), 210 | _compiled_regex(r"require\s*\((.*?)\);?"), 211 | _compiled_regex(r"undefined"), 212 | _compiled_regex(r"\.length"), 213 | _compiled_regex(r"\$\((.*?)\);"), 214 | ], 215 | # C# markers 216 | "c#": [ 217 | # TODO: these are not good 218 | _compiled_regex(r"^\s*#region(.*?)#endregion$"), 219 | _compiled_regex(r"^\s*foreach\s*\((.*?)$", dotall=False), 220 | _compiled_regex(r"^\s*using(.*?)$"), 221 | _compiled_regex(r":\s*base\([^\)]+\)$"), 222 | _compiled_regex(r"base\.\w+"), 223 | _compiled_regex(r"ref\s+\w+"), 224 | _compiled_regex(r"^\s*namespace\s+\w+(\.\w+)+\s*\{(.*?)\};(.*?)$"), 225 | _compiled_regex(r"string\.\w+"), 226 | _compiled_regex(r"///"), 227 | _compiled_regex(r"///\s*<\w+>$"), 228 | _compiled_regex( 229 | r"\[\w+(\.\w+)*\(?[^\]]*\]\s*(public|protected|private|internal|\w+(\s+\w+)*\()(.*?)" 230 | ), 231 | _compiled_regex(r"(sealed\s+)?class\s*\{(.*?)\}"), 232 | _compiled_regex( 233 | r"(sealed\s+)?class\s+\w+(\.\w+)*\s*:\s*\w+(\.\w+)*\s*\{(.*?)\}" 234 | ), 235 | _compiled_regex(r"get\s*{"), 236 | _compiled_regex(r"set\s*{"), 237 | _compiled_regex(r"private\s+get\s*{"), 238 | _compiled_regex(r"private\s+set\s*{"), 239 | ], 240 | # C++ markers 241 | "cpp": [ 242 | _compiled_regex(r"^\s*template\s*<[^>]>$"), 243 | _compiled_regex(r"size_t"), 244 | _compiled_regex(r"\w*\s*::\s*\w+"), 245 | _compiled_regex(r"\w+\s*::\s*\w+\((.*?)\);"), 246 | _compiled_regex(r"\w+\s*::\s*\w+\([^\{]+\s*\{(.*?)\w+::\w+\("), 247 | _compiled_regex(r"(std::)?cout\s*<<(.*?);"), 248 | _compiled_regex(r"(std::)?cin\s*>>(.*?);"), 249 | _compiled_regex(r"std::\w+"), 250 | _compiled_regex(r"std::\w+\((.*?)\)"), 251 | _compiled_regex(r"static_assert\((.*?);"), 252 | _compiled_regex(r"static_cast<[^>]>"), 253 | _compiled_regex(r"dynamic_cast<[^>]>"), 254 | _compiled_regex(r"nullptr"), 255 | _compiled_regex(r"//(.*?)$"), 256 | _compiled_regex(r"switch\s*\((.*?)\);"), 257 | _compiled_regex(r"&\(?\w+"), 258 | _compiled_regex(r"\w+&"), 259 | _compiled_regex(r"\s[A-Z0-9_]+\((.*?);"), 260 | _compiled_regex(r"\)\s*=\s*0;$"), 261 | _compiled_regex(r"~\w+\((.*?)\}"), 262 | _compiled_regex(r"^\s*public:(.*?)};"), 263 | _compiled_regex(r"^\s*private:(.*?)};"), 264 | _compiled_regex(r"^\s*protected:(.*?)};"), 265 | _compiled_regex(r"\sm_\w+"), 266 | _compiled_regex(r"return\s+(.*?);$"), 267 | _compiled_regex(r"^\s*class\s*\w+\s*:\s*public\s+\w+\s*\{(.*?)\)"), 268 | _compiled_regex(r"^\s*virtual\s+[^\(]+\((.*?)\)"), 269 | _compiled_regex(r"^\w*struct\s*(\w+\s*)?{"), 270 | _compiled_regex(r"\w+->\w+"), 271 | _compiled_regex(r"^\s*namespace\s+\w+\s*\{(.*?)\};(.*?)$"), 272 | _compiled_regex(r"const\s+static|static\s+const"), 273 | _compiled_regex(r"typedef\s+(.*?)\s+\w+\s*;$"), 274 | _compiled_regex(r"(i|u)(int)?\d+(_t)?"), 275 | _compiled_regex(r"\*\w+->"), 276 | _compiled_regex(r"(const\s+)?char\s*\*"), 277 | _compiled_regex(r"int\s+\w+"), 278 | _compiled_regex(r"void\s+\w+"), 279 | _compiled_regex(r"auto"), 280 | ], 281 | # Lua markers 282 | "lua": [ 283 | # TODO 284 | _compiled_regex(r"--\[\[(.*?)\]\]"), 285 | _compiled_regex(r"local\s+\w+\s*="), 286 | ], 287 | "php": [ 288 | # PHP markers 289 | _compiled_regex(r"<\?php(.*?)\?>"), 290 | _compiled_regex(r"<\?php"), 291 | _compiled_regex(r"\$\w+"), 292 | _compiled_regex(r"\$\w+\s+=[^;]+;"), 293 | _compiled_regex(r"new\s*\\\w+"), 294 | _compiled_regex(r"\s+\.\s+"), 295 | _compiled_regex(r"this->"), 296 | ], 297 | "ruby": [ 298 | # Ruby markers 299 | _compiled_regex(r"^\s*def\s*[^:]+$(.*?)end$"), 300 | _compiled_regex(r"@[\.:\w+]"), 301 | _compiled_regex(r"\s:\w+"), 302 | _compiled_regex(r"#\{(.*?)\}"), 303 | _compiled_regex(r"^\s*include\s+[\.\w+]+$"), 304 | _compiled_regex(r"^\s*alias\s[\.\w]+\s+[\.\w]+(.*?)$"), 305 | _compiled_regex(r"^\s*class\s+[\.\w+]+(\s*<\s*[\.\w]+(::[\.\w]+)*)?(.*?)$"), 306 | _compiled_regex(r"^\s*module\s+[\.\w+]+\s*[\.\w]+(::[\.\w]+)*(.*?)$"), 307 | ], 308 | # Java markers 309 | "java": [ 310 | _compiled_regex(r"\sstatic\s+final\s"), 311 | _compiled_regex(r"(public|protected|private)\s+synchronized\s"), 312 | _compiled_regex(r"synchronized\s*\([^\{]+\{(.*?)\}"), 313 | _compiled_regex(r"ArrayList<[\.\w+]*>"), 314 | _compiled_regex(r"HashMap<[\.\w+]*>"), 315 | _compiled_regex(r"HashSet<[\.\w+]*>"), 316 | _compiled_regex(r"System(\.\w+)+"), 317 | _compiled_regex(r"new\s+\w+(.*?);"), 318 | _compiled_regex(r"try\s*\{(.*?)catch[^\{]+\{"), 319 | _compiled_regex(r"[Ll]ogg(ing|er)"), 320 | _compiled_regex(r"^\s*package\s+\w+(\.\w+)*;$"), 321 | _compiled_regex(r"^\s*import\s+\w+(\.\w+)*;$"), 322 | _compiled_regex(r"(public|private|protected)\s+[^\{]*\{(.*?)\}$"), 323 | _compiled_regex(r"@Override"), 324 | _compiled_regex(r"throw new \w+\((.*?)\);\s*$"), 325 | ], 326 | # TeX markers 327 | "tex": [ 328 | _compiled_regex(r"\\begin\{[^\}]+\}(.*?)\\end\{[^\}]+\}"), 329 | _compiled_regex(r"\\begin\{[^\}]+\}(.*?)*$", dotall=False), 330 | _compiled_regex(r"\\end\{[^\}]+\}(.*?)*$", dotall=False), 331 | _compiled_regex(r"\\\w+\s"), 332 | _compiled_regex(r"\\\w+({|\[)"), 333 | _compiled_regex(r"\\usepackage(\[[^\]]*\])?\{(.*?)\}"), 334 | _compiled_regex(r"^\s*%.{15,}$", dotall=False), 335 | ], 336 | # Make markers 337 | # MDM: trying to find out why make gets over reported. 338 | # MDM: trying to find out why make gets over reported. 339 | "make": [ 340 | # _compiled_regex(r"^\s*[^=\n]+\s*=\s*(.*?)$"), 341 | # _compiled_regex(r"^\s*[^=\n]+\s*:=\s*(.*?)$"), 342 | # _compiled_regex(r"^\s*[^=\n]+\s*\+=\s*(.*?)$"), 343 | # _compiled_regex(r"^\s*[^=\n]+\s*\?=\s*(.*?)$"), 344 | # _compiled_regex(r"\$@|\$\*|\$^|\$<"), 345 | # _compiled_regex(r"\$\((.*?)\)", dotall=False), 346 | # _compiled_regex(r"\w+\.\w{1,3}"), 347 | # _compiled_regex(r"^\s*ifeq[^\n]+(.*?)^\s*endif"), 348 | # _compiled_regex(r"^\s*ifneq[^\n]+(.*?)^\s*endif"), 349 | # _compiled_regex(r"\w*%\w*"), 350 | # _compiled_regex(r"-[^\W\d]+"), 351 | _compiled_regex(r"PHONY\s*\+=(.*?)$"), 352 | _compiled_regex(r"\.PHONY:(.*?)$"), 353 | ], 354 | } 355 | -------------------------------------------------------------------------------- /whats_that_code/known_languages.py: -------------------------------------------------------------------------------- 1 | """ 2 | Database of known languages 3 | 4 | TODO: refactor for alternate extensions 5 | """ 6 | 7 | # because one is a subset of the other 8 | CLONES = [["xml", "html", "php"], ["c", "c++"], ["javascript", "typescript"]] 9 | 10 | # popularity data is from: 11 | # https://pypl.github.io/PYPL.html?country=US 12 | # CC-BY-SA 3.0, original data from google. 13 | POPULARITY = { 14 | "python": "1", 15 | "java": "2", 16 | "javascript": "3", 17 | "c#": "4", 18 | "c++": "5", 19 | "php": "6", 20 | "r": "7", 21 | "objectivec": "8", 22 | "swift": "9", 23 | "typescript": "10", 24 | "matlab": "11", 25 | "kotlin": "12", 26 | "go": "13", 27 | "vba": "14", 28 | "ruby": "15", 29 | "rust": "16", 30 | "scala": "17", 31 | "vbnet": "18", 32 | "lua": "19", 33 | "ada": "20", 34 | "dart": "21", 35 | "abap": "22", 36 | "perl": "23", 37 | "julia": "24", 38 | "groovy": "25", 39 | "cobol": "26", 40 | "haskell": "27", 41 | "delphi": "28", 42 | } 43 | POPULARITY_LIST = [ 44 | "python", 45 | "java", 46 | "javascript", 47 | "c#", 48 | "c++", 49 | "php", 50 | "r", 51 | "objectivec", 52 | "swift", 53 | "typescript", 54 | "matlab", 55 | "kotlin", 56 | "go", 57 | "vba", 58 | "ruby", 59 | "rust", 60 | "scala", 61 | "vbnet", 62 | "lua", 63 | "ada", 64 | "dart", 65 | "abap", 66 | "perl", 67 | "julia", 68 | "groovy", 69 | "cobol", 70 | "haskell", 71 | "delphi", 72 | ] 73 | 74 | FILE_EXTENSIONS = { 75 | "batchfile": [".bat"], 76 | # "c": [".c"], 77 | "c#": [".cs"], 78 | "c++": [".cpp", ".gcc"], 79 | # "css": [".css"], 80 | # "coffeescript": [".coffee"], 81 | # "erlang": [".erlang"], 82 | # "go": [".go"], 83 | # "html": [".html"], 84 | # "haskell": [".haskell"], 85 | # "java": [".java"], 86 | # "javascript": [".js"], 87 | # "json": [".json"], 88 | "jupyter notebook": [".nb"], 89 | # "lua": [".lua"], 90 | # "markdown": [".md"], 91 | # "matlab": [".matlab"], 92 | "objective-C": [".objc"], 93 | # "php": [".php"], 94 | # "perl": [".perl"], 95 | # "powershell": [".ps1"], 96 | # "python": [".py"], 97 | "r": [".r"], 98 | # "ruby": [".rb"], 99 | # "rust": [".rust"], 100 | # "sql": [".sql"], 101 | # "scala": [".scala"], 102 | "shell": [".sh"], 103 | # "swift": [".swift"], 104 | # "tex": [".tex"], 105 | # "yaml": [".yml"], 106 | # "typescript": [".ts"], 107 | "abap": [".abap", ".ABAP"], 108 | "apl": [".apl"], 109 | "abnf": [".abnf"], 110 | "actionscript3": [".as"], 111 | "actionscript": [".as"], 112 | "ada": [".adb", ".ads", ".ada"], 113 | "adl": [".adl", ".adls", ".adlf", ".adlx"], 114 | "agda": [".agda"], 115 | "aheui": [".aheui"], 116 | "alloy": [".als"], 117 | "ambienttalk": [".at"], 118 | "ampl": [".run"], 119 | "angular2html": [".ng2"], 120 | "antlractionscript": [".G", ".g"], 121 | "antlrcsharp": [".G", ".g"], 122 | "antlrcpp": [".G", ".g"], 123 | "antlrjava": [".G", ".g"], 124 | "antlrobjectivec": [".G", ".g"], 125 | "antlrperl": [".G", ".g"], 126 | "antlrpython": [".G", ".g"], 127 | "antlrruby": [".G", ".g"], 128 | "apacheconf": [".htaccess", "apache.conf", "apache2.conf"], 129 | "applescript": [".applescript"], 130 | "arduino": [".ino"], 131 | "arrow": [".arw"], 132 | "aspectj": [".aj"], 133 | "asymptote": [".asy"], 134 | "augeas": [".aug"], 135 | "autoit": [".au3"], 136 | "autohotkey": [".ahk", ".ahkl"], 137 | "awk": [".awk"], 138 | "bbcbasic": [".bbc"], 139 | "bc": [".bc"], 140 | "bst": [".bst"], 141 | "bare": [".bare"], 142 | "bash": [ 143 | ".sh", 144 | ".ksh", 145 | ".bash", 146 | ".ebuild", 147 | ".eclass", 148 | ".exheres-0", 149 | ".exlib", 150 | ".zsh", 151 | ".bashrc", 152 | "bashrc", 153 | ".bash_*", 154 | "bash_*", 155 | "zshrc", 156 | ".zshrc", 157 | "PKGBUILD", 158 | ], 159 | "bashsession": [".sh-session", ".shell-session"], 160 | "batch": [".bat", ".cmd"], 161 | "befunge": [".befunge"], 162 | "bibtex": [".bib"], 163 | "blitzbasic": [".bb", ".decls"], 164 | "blitzmax": [".bmx"], 165 | "bnf": [".bnf"], 166 | "boa": [".boa"], 167 | "boo": [".boo"], 168 | "boogie": [".bpl"], 169 | "brainfuck": [".bf", ".b"], 170 | "bugs": [".bug"], 171 | "camkes": [".camkes", ".idl4"], 172 | "c": [".c", ".h", ".idc"], 173 | "cmake": [".cmake", "CMakeLists.txt"], 174 | "cobjdump": [".c-objdump"], 175 | "cpsa": [".cpsa"], 176 | "csharpaspx": [".aspx", ".asax", ".ascx", ".ashx", ".asmx", ".axd"], 177 | "csharp": [".cs"], 178 | "ca65": [".s"], 179 | "cadl": [".cadl"], 180 | "capdl": [".cdl"], 181 | "capnproto": [".capnp"], 182 | "cbmbasicv2": [".bas"], 183 | "ceylon": [".ceylon"], 184 | "cfengine3": [".cf"], 185 | "chaiscript": [".chai"], 186 | "chapel": [".chpl"], 187 | "charmci": [".ci"], 188 | "cheetah": [".tmpl", ".spt"], 189 | "cirru": [".cirru"], 190 | "clay": [".clay"], 191 | "clean": [".icl", ".dcl"], 192 | "clojure": [".clj"], 193 | "clojurescript": [".cljs"], 194 | "cobolfreeformat": [".cbl", ".CBL"], 195 | "cobol": [".cob", ".COB", ".cpy", ".CPY"], 196 | "coffeescript": [".coffee"], 197 | "coldfusioncfc": [".cfc"], 198 | "coldfusionhtml": [".cfm", ".cfml"], 199 | "commonlisp": [".cl", ".lisp"], 200 | "componentpascal": [".cp", ".cps"], 201 | "coq": [".v"], 202 | "cpp": [ 203 | ".cpp", 204 | ".hpp", 205 | ".c++", 206 | ".h++", 207 | ".cc", 208 | ".hh", 209 | ".cxx", 210 | ".hxx", 211 | ".C", 212 | ".H", 213 | ".cp", 214 | ".CPP", 215 | ], 216 | "cppobjdump": [".cpp-objdump", ".c++-objdump", ".cxx-objdump"], 217 | "crmsh": [".crmsh", ".pcmk"], 218 | "croc": [".croc"], 219 | "cryptol": [".cry"], 220 | "crystal": [".cr"], 221 | "csounddocument": [".csd"], 222 | "csoundorchestra": [".orc", ".udo"], 223 | "csoundscore": [".sco"], 224 | "css": [".css"], 225 | "cuda": [".cu", ".cuh"], 226 | "cypher": [".cyp", ".cypher"], 227 | "cython": [".pyx", ".pxd", ".pxi"], 228 | "d": [".d", ".di"], 229 | "dobjdump": [".d-objdump"], 230 | "darcspatch": [".dpatch", ".darcspatch"], 231 | "dart": [".dart"], 232 | "dasm16": [".dasm16", ".dasm"], 233 | "debiancontrol": ["control"], 234 | "delphi": [".pas", ".dpr"], 235 | "devicetree": [".dts", ".dtsi"], 236 | "dg": [".dg"], 237 | "diff": [".diff", ".patch"], 238 | "docker": ["Dockerfile", ".docker"], 239 | "dtd": [".dtd"], 240 | "duel": [".duel", ".jbst"], 241 | "dylanconsole": [".dylan-console"], 242 | "dylan": [".dylan", ".dyl", ".intr"], 243 | "dylanlid": [".lid", ".hdp"], 244 | "ecl": [".ecl"], 245 | "ec": [".ec", ".eh"], 246 | "earlgrey": [".eg"], 247 | "easytrieve": [".ezt", ".mac"], 248 | "ebnf": [".ebnf"], 249 | "eiffel": [".e"], 250 | "elixir": [".ex", ".eex", ".exs"], 251 | "elm": [".elm"], 252 | "emacslisp": [".el"], 253 | # "email": [".eml"], # not a programming language 254 | "erlang": [".erl", ".hrl", ".es", ".escript"], 255 | "erlangshell": [".erl-sh"], 256 | "evoquehtml": [".html"], 257 | "evoque": [".evoque"], 258 | "evoquexml": [".xml"], 259 | "execline": [".exec"], 260 | "ezhil": [".n"], 261 | "fsharp": [".fs", ".fsi"], 262 | "fstar": [".fst", ".fsti"], 263 | "factor": [".factor"], 264 | "fancy": [".fy", ".fancypack"], 265 | "fantom": [".fan"], 266 | "felix": [".flx", ".flxh"], 267 | "fennel": [".fnl"], 268 | "fishshell": [".fish", ".load"], 269 | "floscript": [".flo"], 270 | "forth": [".frt", ".fs"], 271 | "fortranfixed": [".f", ".F"], 272 | "fortran": [".f03", ".f90", ".F03", ".F90"], 273 | "foxpro": [".PRG", ".prg"], 274 | "freefem": [".edp"], 275 | "gap": [".g", ".gd", ".gi", ".gap"], 276 | "gdscript": [".gd"], 277 | "glshader": [".vert", ".frag", ".geo"], 278 | "gas": [".s", ".S"], 279 | "genshi": [".kid"], 280 | "gettext": [".pot", ".po"], 281 | "gherkin": [".feature"], 282 | "gnuplot": [".plot", ".plt"], 283 | "go": [".go"], 284 | "golo": [".golo"], 285 | "gooddatacl": [".gdc"], 286 | "gosu": [".gs", ".gsx", ".gsp", ".vark"], 287 | "gosutemplate": [".gst"], 288 | "groff": [".[1234567]", ".man"], 289 | "groovy": [".groovy", ".gradle"], 290 | "hlslshader": [".hlsl", ".hlsli"], 291 | "haml": [".haml"], 292 | "handlebarshtml": [".handlebars", ".hbs"], 293 | "haskell": [".hs"], 294 | "haxe": [".hx", ".hxsl"], 295 | "hsail": [".hsail"], 296 | "html": [".html", ".htm", ".xhtml", ".xslt"], 297 | "htmlphp": [".phtml"], 298 | "hxml": [".hxml"], 299 | "hy": [".hy"], 300 | "hybris": [".hy", ".hyb"], 301 | "idl": [".pro"], 302 | "icon": [".icon", ".ICON"], 303 | "idris": [".idr"], 304 | "igor": [".ipf"], 305 | "inform6": [".inf"], 306 | "inform6template": [".i6t"], 307 | "inform7": [".ni", ".i7x"], 308 | "ini": [".ini", ".cfg", ".inf"], 309 | "io": [".io"], 310 | "ioke": [".ik"], 311 | "irclogs": [".weechatlog"], 312 | "isabelle": [".thy"], 313 | "j": [".ijs"], 314 | "jags": [".jag", ".bug"], 315 | "jasmin": [".j"], 316 | "java": [".java"], 317 | "javascript": [".js", ".jsm", ".mjs", ".javascript"], 318 | "jcl": [".jcl"], 319 | "jsgf": [".jsgf"], 320 | "jsonld": [".jsonld"], 321 | "json": [".json", "Pipfile.lock"], 322 | "jsp": [".jsp"], 323 | "julia": [".jl"], 324 | "juttle": [".juttle"], 325 | "kal": [".kal"], 326 | "kconfig": ["Kconfig*", "*Config.in*", "external.in*", "standard-modules.in"], 327 | "kernellog": [".kmsg", ".dmesg"], 328 | "koka": [".kk", ".kki"], 329 | "kotlin": [".kt", ".kts"], 330 | "lsl": [".lsl"], 331 | "lasso": [".lasso", ".lasso[89]"], 332 | "lean": [".lean"], 333 | "lesscss": [".less"], 334 | "limbo": [".b"], 335 | "liquid": [".liquid"], 336 | "literateagda": [".lagda"], 337 | "literatecryptol": [".lcry"], 338 | "literatehaskell": [".lhs"], 339 | "literateidris": [".lidr"], 340 | "livescript": [".ls"], 341 | "llvm": [".ll"], 342 | "llvmmir": [".mir"], 343 | "logos": [".x", ".xi", ".xm", ".xmi"], 344 | "logtalk": [".lgt", ".logtalk"], 345 | "lua": [".lua", ".wlua"], 346 | "moocode": [".moo"], 347 | "makefile": [".mak", ".mk", "Makefile", "makefile", "Makefile.*", "GNUmakefile"], 348 | "mako": [".mao"], 349 | "maql": [".maql"], 350 | "markdown": [".md", ".markdown"], 351 | "mask": [".mask"], 352 | "mason": [".m", ".mhtml", ".mc", ".mi", "autohandler", "dhandler"], 353 | "mathematica": [".nb", ".cdf", ".nbp", ".ma"], 354 | "matlab": [".m"], 355 | "miniscript": [".ms"], 356 | "modelica": [".mo"], 357 | "modula2": [".def", ".mod"], 358 | "monkey": [".monkey"], 359 | "monte": [".mt"], 360 | "moonscript": [".moon"], 361 | "mosel": [".mos"], 362 | "mozpreproccss": [".css.in"], 363 | "mozpreprocjavascript": [".js.in"], 364 | "mozpreprocxul": [".xul.in"], 365 | "mql": [".mq4", ".mq5", ".mqh"], 366 | "mscgen": [".msc"], 367 | "mupad": [".mu"], 368 | "mxml": [".mxml"], 369 | "myghty": [".myt", "autodelegate"], 370 | "ncl": [".ncl"], 371 | "nsis": [".nsi", ".nsh"], 372 | "nasm": [".asm", ".ASM"], 373 | "nasmobjdump": [".objdump-intel"], 374 | "nemerle": [".n"], 375 | "nesc": [".nc"], 376 | "newlisp": [".lsp", ".nl", ".kif"], 377 | "newspeak": [".ns2"], 378 | "nginxconf": ["nginx.conf"], 379 | "nimrod": [".nim", ".nimrod"], 380 | "nit": [".nit"], 381 | "nix": [".nix"], 382 | "nusmv": [".smv"], 383 | "objdump": [".objdump"], 384 | "objectivec": [".m", ".h"], 385 | "objectivecpp": [".mm", ".hh"], 386 | "objectivej": [".j"], 387 | "ocaml": [".ml", ".mli", ".mll", ".mly"], 388 | "octave": [".m"], 389 | "odin": [".odin"], 390 | "ooc": [".ooc"], 391 | "opa": [".opa"], 392 | "openedge": [".p", ".cls"], 393 | "pacmanconf": ["pacman.conf"], 394 | "pan": [".pan"], 395 | "parasail": [".psi", ".psl"], 396 | "pawn": [".p", ".pwn", ".inc"], 397 | "peg": [".peg"], 398 | "perl6": [ 399 | ".pl", 400 | ".pm", 401 | ".nqp", 402 | ".p6", 403 | ".6pl", 404 | ".p6l", 405 | ".pl6", 406 | ".6pm", 407 | ".p6m", 408 | ".pm6", 409 | ".t", 410 | ".raku", 411 | ".rakumod", 412 | ".rakutest", 413 | ".rakudoc", 414 | ], 415 | "perl": [".pl", ".pm", ".t", ".perl"], 416 | "php": [".php", ".php[345]", ".inc"], 417 | "pig": [".pig"], 418 | "pike": [".pike", ".pmod"], 419 | "pkgconfig": [".pc"], 420 | "pointless": [".ptls"], 421 | "pony": [".pony"], 422 | "postscript": [".ps", ".eps"], 423 | "povray": [".pov", ".inc"], 424 | "powershell": [".ps1", ".psm1"], 425 | "praat": [".praat", ".proc", ".psc"], 426 | "prolog": [".ecl", ".prolog", ".pro", ".pl"], 427 | "promql": [".promql"], 428 | "properties": [".properties"], 429 | "protobuf": [".proto"], 430 | "pug": [".pug", ".jade"], 431 | "puppet": [".pp"], 432 | "pypylog": [".pypylog"], 433 | "python2traceback": [".py2tb"], 434 | "python": [ 435 | ".py", 436 | ".pyw", 437 | ".jy", 438 | ".sage", 439 | ".sc", 440 | "SConstruct", 441 | "SConscript", 442 | ".bzl", 443 | "BUCK", 444 | "BUILD", 445 | "BUILD.bazel", 446 | "WORKSPACE", 447 | ".tac", 448 | ".python", 449 | ], 450 | "pythontraceback": [".pytb", ".py3tb"], 451 | "qbasic": [".BAS", ".bas"], 452 | "qvto": [".qvto"], 453 | "qml": [".qml", ".qbs"], 454 | "rconsole": [".Rout"], 455 | "rnccompact": [".rnc"], 456 | "rpmspec": [".spec"], 457 | "racket": [".rkt", ".rktd", ".rktl"], 458 | "ragelc": [".rl"], 459 | "ragelcpp": [".rl"], 460 | "rageld": [".rl"], 461 | "ragelembedded": [".rl"], 462 | "rageljava": [".rl"], 463 | "ragelobjectivec": [".rl"], 464 | "ragelruby": [".rl"], 465 | "rd": [".Rd"], 466 | "reason": [".re", ".rei"], 467 | "rebol": [".r", ".r3", ".reb"], 468 | "red": [".red", ".reds"], 469 | "redcode": [".cw"], 470 | "regedit": [".reg"], 471 | "rexx": [".rexx", ".rex", ".rx", ".arexx"], 472 | "rhtml": [".rhtml"], 473 | "ride": [".ride"], 474 | "roboconfgraph": [".graph"], 475 | "roboconfinstances": [".instances"], 476 | "robotframework": [".robot"], 477 | "rql": [".rql"], 478 | "rsl": [".rsl"], 479 | "rst": [".rst", ".rest"], 480 | "rts": [".rts"], 481 | "ruby": [ 482 | ".rb", 483 | ".rbw", 484 | "Rakefile", 485 | ".rake", 486 | ".gemspec", 487 | ".rbx", 488 | ".duby", 489 | "Gemfile", 490 | ".ruby", 491 | ], 492 | "rust": [".rs", ".rs.in"], 493 | "sas": [".SAS", ".sas"], 494 | "s": [".S", ".R", ".Rhistory", ".Rprofile", ".Renviron"], 495 | "sml": [".sml", ".sig", ".fun"], 496 | "sarl": [".sarl"], 497 | "sass": [".sass"], 498 | "scala": [".scala"], 499 | "scaml": [".scaml"], 500 | "scdoc": [".scd", ".scdoc"], 501 | "scheme": [".scm", ".ss"], 502 | "scilab": [".sci", ".sce", ".tst"], 503 | "scss": [".scss"], 504 | "shexc": [".shex"], 505 | "shen": [".shen"], 506 | "sieve": [".siv", ".sieve"], 507 | "silver": [".sil", ".vpr"], 508 | "singularity": [".def", "Singularity"], 509 | "slash": [".sla"], 510 | "slim": [".slim"], 511 | "slurmbash": [".sl"], 512 | "smali": [".smali"], 513 | "smalltalk": [".st"], 514 | "smartgameformat": [".sgf"], 515 | "smarty": [".tpl"], 516 | "snobol": [".snobol"], 517 | "snowball": [".sbl"], 518 | "solidity": [".sol"], 519 | "sourcepawn": [".sp"], 520 | "sourceslist": ["sources.list"], 521 | "sparql": [".rq", ".sparql"], 522 | "sql": [".sql"], 523 | "sqliteconsole": [".sqlite3-console"], 524 | "squidconf": ["squid.conf"], 525 | "ssp": [".ssp"], 526 | "stan": [".stan"], 527 | "stata": [".do", ".ado"], 528 | "supercollider": [".sc", ".scd"], 529 | "swift": [".swift"], 530 | "swig": [".swg", ".i"], 531 | "systemverilog": [".sv", ".svh"], 532 | "tap": [".tap"], 533 | "tnt": [".tnt"], 534 | "toml": [".toml", "Pipfile", "poetry.lock"], 535 | "tads3": [".t"], 536 | "tasm": [".asm", ".ASM", ".tasm"], 537 | "tcl": [".tcl", ".rvt"], 538 | "tcsh": [".tcsh", ".csh"], 539 | "teatemplate": [".tea"], 540 | "teraterm": [".ttl"], 541 | "termcap": ["termcap", "termcap.src"], 542 | "terminfo": ["terminfo", "terminfo.src"], 543 | "terraform": [".tf"], 544 | "tex": [".tex", ".aux", ".toc"], 545 | "text": [".txt"], 546 | "thrift": [".thrift"], 547 | "tiddlywiki5": [".tid"], 548 | "todotxt": ["todo.txt", ".todotxt"], 549 | "transactsql": [".sql"], 550 | "treetop": [".treetop", ".tt"], 551 | "turtle": [".ttl"], 552 | "twightml": [".twig"], 553 | "typescript": [".ts", ".tsx"], 554 | "typoscript": [".typoscript"], 555 | "ucode": [".u", ".u1", ".u2"], 556 | "unicon": [".icn"], 557 | "urbiscript": [".u"], 558 | "usd": [".usd", ".usda"], 559 | "vbscript": [".vbs", ".VBS"], 560 | "vcl": [".vcl"], 561 | "vgl": [".rpf"], 562 | "vala": [".vala", ".vapi"], 563 | "vbnetaspx": [".aspx", ".asax", ".ascx", ".ashx", ".asmx", ".axd"], 564 | "vbnet": [".vb", ".bas"], 565 | "velocity": [".vm", ".fhtml"], 566 | "verilog": [".v"], 567 | "vhdl": [".vhdl", ".vhd"], 568 | "vim": [ 569 | ".vim", 570 | ".vimrc", 571 | ".exrc", 572 | ".gvimrc", 573 | "_vimrc", 574 | "_exrc", 575 | "_gvimrc", 576 | "vimrc", 577 | "gvimrc", 578 | ], 579 | "wdiff": [".wdiff"], 580 | "webidl": [".webidl"], 581 | "whiley": [".whiley"], 582 | "x10": [".x10"], 583 | "xquery": [".xqy", ".xquery", ".xq", ".xql", ".xqm"], 584 | "xml": [".xml", ".xsl", ".rss", ".xslt", ".xsd", ".wsdl", ".wsf"], 585 | "xorg": ["xorg.conf"], 586 | "xslt": [".xsl", ".xslt", ".xpl"], 587 | "xtend": [".xtend"], 588 | "xtlang": [".xtm"], 589 | "yamljinja": [".sls"], 590 | "yaml": [".yaml", ".yml"], 591 | "yang": [".yang"], 592 | "zeek": [".zeek", ".bro"], 593 | "zephir": [".zep"], 594 | "zig": [".zig"], 595 | } 596 | -------------------------------------------------------------------------------- /.config/.pylintrc: -------------------------------------------------------------------------------- 1 | [MAIN] 2 | 3 | # Analyse import fallback blocks. This can be used to support both Python 2 and 4 | # 3 compatible code, which means that the block might have code that exists 5 | # only in one or another interpreter, leading to false positives when analysed. 6 | analyse-fallback-blocks=no 7 | 8 | # Load and enable all available extensions. Use --list-extensions to see a list 9 | # all available extensions. 10 | #enable-all-extensions= 11 | 12 | # In error mode, checkers without error messages are disabled and for others, 13 | # only the ERROR messages are displayed, and no reports are done by default. 14 | #errors-only= 15 | 16 | # Always return a 0 (non-error) status code, even if lint errors are found. 17 | # This is primarily useful in continuous integration scripts. 18 | #exit-zero= 19 | 20 | # A comma-separated list of package or module names from where C extensions may 21 | # be loaded. Extensions are loading into the active Python interpreter and may 22 | # run arbitrary code. 23 | extension-pkg-allow-list= 24 | 25 | # A comma-separated list of package or module names from where C extensions may 26 | # be loaded. Extensions are loading into the active Python interpreter and may 27 | # run arbitrary code. (This is an alternative name to extension-pkg-allow-list 28 | # for backward compatibility.) 29 | extension-pkg-whitelist= 30 | 31 | # Return non-zero exit code if any of these messages/categories are detected, 32 | # even if score is above --fail-under value. Syntax same as enable. Messages 33 | # specified are enabled, while categories only check already-enabled messages. 34 | fail-on= 35 | 36 | # Specify a score threshold to be exceeded before program exits with error. 37 | fail-under=10 38 | 39 | # Interpret the stdin as a python script, whose filename needs to be passed as 40 | # the module_or_package argument. 41 | #from-stdin= 42 | 43 | # Files or directories to be skipped. They should be base names, not paths. 44 | ignore=CVS 45 | 46 | # Add files or directories matching the regex patterns to the ignore-list. The 47 | # regex matches against paths and can be in Posix or Windows format. 48 | ignore-paths= 49 | 50 | # Files or directories matching the regex patterns are skipped. The regex 51 | # matches against base names, not paths. The default value ignores Emacs file 52 | # locks 53 | ignore-patterns=^\.# 54 | 55 | # List of module names for which member attributes should not be checked 56 | # (useful for modules/projects where namespaces are manipulated during runtime 57 | # and thus existing member attributes cannot be deduced by static analysis). It 58 | # supports qualified module names, as well as Unix pattern matching. 59 | ignored-modules= 60 | 61 | # Python code to execute, usually for sys.path manipulation such as 62 | # pygtk.require(). 63 | #init-hook= 64 | 65 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 66 | # number of processors available to use. 67 | jobs=1 68 | 69 | # Control the amount of potential inferred values when inferring a single 70 | # object. This can help the performance when dealing with large functions or 71 | # complex, nested conditions. 72 | limit-inference-results=100 73 | 74 | # List of plugins (as comma separated values of python module names) to load, 75 | # usually to register additional checkers. 76 | load-plugins= 77 | 78 | # Pickle collected data for later comparisons. 79 | persistent=yes 80 | 81 | # Minimum Python version to use for version dependent checks. Will default to 82 | # the version used to run pylint. 83 | py-version=3.10 84 | 85 | # Discover python modules and packages in the file system subtree. 86 | recursive=no 87 | 88 | # When enabled, pylint would attempt to guess common misconfiguration and emit 89 | # user-friendly hints instead of false-positive error messages. 90 | suggestion-mode=yes 91 | 92 | # Allow loading of arbitrary C extensions. Extensions are imported into the 93 | # active Python interpreter and may run arbitrary code. 94 | unsafe-load-any-extension=no 95 | 96 | # In verbose mode, extra non-checker-related info will be displayed. 97 | #verbose= 98 | 99 | 100 | [REPORTS] 101 | 102 | # Python expression which should return a score less than or equal to 10. You 103 | # have access to the variables 'fatal', 'error', 'warning', 'refactor', 104 | # 'convention', and 'info' which contain the number of messages in each 105 | # category, as well as 'statement' which is the total number of statements 106 | # analyzed. This score is used by the global evaluation report (RP0004). 107 | evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) 108 | 109 | # Template used to display messages. This is a python new-style format string 110 | # used to format the message information. See doc for all details. 111 | msg-template= 112 | 113 | # Set the output format. Available formats are text, parseable, colorized, json 114 | # and msvs (visual studio). You can also give a reporter class, e.g. 115 | # mypackage.mymodule.MyReporterClass. 116 | #output-format= 117 | 118 | # Tells whether to display a full report or only the messages. 119 | reports=no 120 | 121 | # Activate the evaluation score. 122 | score=yes 123 | 124 | 125 | [MESSAGES CONTROL] 126 | 127 | # Only show warnings with the listed confidence levels. Leave empty to show 128 | # all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, 129 | # UNDEFINED. 130 | confidence=HIGH, 131 | CONTROL_FLOW, 132 | INFERENCE, 133 | INFERENCE_FAILURE, 134 | UNDEFINED 135 | 136 | # Disable the message, report, category or checker with the given id(s). You 137 | # can either give multiple identifiers separated by comma (,) or put this 138 | # option multiple times (only on the command line, not in the configuration 139 | # file where it should appear only once). You can also use "--disable=all" to 140 | # disable everything first and then re-enable specific checks. For example, if 141 | # you want to run only the similarities checker, you can use "--disable=all 142 | # --enable=similarities". If you want to run only the classes checker, but have 143 | # no Warning level messages displayed, use "--disable=all --enable=classes 144 | # --disable=W". 145 | disable=raw-checker-failed, 146 | bad-inline-option, 147 | locally-disabled, 148 | file-ignored, 149 | suppressed-message, 150 | useless-suppression, 151 | deprecated-pragma, 152 | use-symbolic-message-instead, 153 | 154 | fixme, 155 | too-many-lines, 156 | too-many-branches, 157 | import-error, 158 | too-many-statements, 159 | useless-import-alias, 160 | too-many-locals, 161 | consider-using-from-import, # pylint is wrong 162 | duplicate-code, 163 | too-many-arguments, 164 | logging-fstring-interpolation, 165 | too-many-instance-attributes, 166 | too-many-return-statements, 167 | unnecessary-direct-lambda-call, 168 | too-few-public-methods, 169 | logging-not-lazy, 170 | too-few-public-methods 171 | 172 | # Enable the message, report, category or checker with the given id(s). You can 173 | # either give multiple identifier separated by comma (,) or put this option 174 | # multiple time (only on the command line, not in the configuration file where 175 | # it should appear only once). See also the "--disable" option for examples. 176 | enable=c-extension-no-member 177 | 178 | 179 | [BASIC] 180 | 181 | # Naming style matching correct argument names. 182 | argument-naming-style=snake_case 183 | 184 | # Regular expression matching correct argument names. Overrides argument- 185 | # naming-style. If left empty, argument names will be checked with the set 186 | # naming style. 187 | #argument-rgx= 188 | 189 | # Naming style matching correct attribute names. 190 | attr-naming-style=snake_case 191 | 192 | # Regular expression matching correct attribute names. Overrides attr-naming- 193 | # style. If left empty, attribute names will be checked with the set naming 194 | # style. 195 | #attr-rgx= 196 | 197 | # Bad variable names which should always be refused, separated by a comma. 198 | bad-names=foo, 199 | bar, 200 | baz, 201 | toto, 202 | tutu, 203 | tata 204 | 205 | # Bad variable names regexes, separated by a comma. If names match any regex, 206 | # they will always be refused 207 | bad-names-rgxs= 208 | 209 | # Naming style matching correct class attribute names. 210 | class-attribute-naming-style=any 211 | 212 | # Regular expression matching correct class attribute names. Overrides class- 213 | # attribute-naming-style. If left empty, class attribute names will be checked 214 | # with the set naming style. 215 | #class-attribute-rgx= 216 | 217 | # Naming style matching correct class constant names. 218 | class-const-naming-style=UPPER_CASE 219 | 220 | # Regular expression matching correct class constant names. Overrides class- 221 | # const-naming-style. If left empty, class constant names will be checked with 222 | # the set naming style. 223 | #class-const-rgx= 224 | 225 | # Naming style matching correct class names. 226 | class-naming-style=PascalCase 227 | 228 | # Regular expression matching correct class names. Overrides class-naming- 229 | # style. If left empty, class names will be checked with the set naming style. 230 | #class-rgx= 231 | 232 | # Naming style matching correct constant names. 233 | const-naming-style=UPPER_CASE 234 | 235 | # Regular expression matching correct constant names. Overrides const-naming- 236 | # style. If left empty, constant names will be checked with the set naming 237 | # style. 238 | #const-rgx= 239 | 240 | # Minimum line length for functions/classes that require docstrings, shorter 241 | # ones are exempt. 242 | docstring-min-length=-1 243 | 244 | # Naming style matching correct function names. 245 | function-naming-style=snake_case 246 | 247 | # Regular expression matching correct function names. Overrides function- 248 | # naming-style. If left empty, function names will be checked with the set 249 | # naming style. 250 | #function-rgx= 251 | 252 | # Good variable names which should always be accepted, separated by a comma. 253 | good-names=i, 254 | j, 255 | k, 256 | ex, 257 | Run, 258 | _ 259 | 260 | # Good variable names regexes, separated by a comma. If names match any regex, 261 | # they will always be accepted 262 | good-names-rgxs= 263 | 264 | # Include a hint for the correct naming format with invalid-name. 265 | include-naming-hint=no 266 | 267 | # Naming style matching correct inline iteration names. 268 | inlinevar-naming-style=any 269 | 270 | # Regular expression matching correct inline iteration names. Overrides 271 | # inlinevar-naming-style. If left empty, inline iteration names will be checked 272 | # with the set naming style. 273 | #inlinevar-rgx= 274 | 275 | # Naming style matching correct method names. 276 | method-naming-style=snake_case 277 | 278 | # Regular expression matching correct method names. Overrides method-naming- 279 | # style. If left empty, method names will be checked with the set naming style. 280 | #method-rgx= 281 | 282 | # Naming style matching correct module names. 283 | module-naming-style=snake_case 284 | 285 | # Regular expression matching correct module names. Overrides module-naming- 286 | # style. If left empty, module names will be checked with the set naming style. 287 | #module-rgx= 288 | 289 | # Colon-delimited sets of names that determine each other's naming style when 290 | # the name regexes allow several styles. 291 | name-group= 292 | 293 | # Regular expression which should only match function or class names that do 294 | # not require a docstring. 295 | no-docstring-rgx=^_ 296 | 297 | # List of decorators that produce properties, such as abc.abstractproperty. Add 298 | # to this list to register other decorators that produce valid properties. 299 | # These decorators are taken in consideration only for invalid-name. 300 | property-classes=abc.abstractproperty 301 | 302 | # Regular expression matching correct type variable names. If left empty, type 303 | # variable names will be checked with the set naming style. 304 | #typevar-rgx= 305 | 306 | # Naming style matching correct variable names. 307 | variable-naming-style=snake_case 308 | 309 | # Regular expression matching correct variable names. Overrides variable- 310 | # naming-style. If left empty, variable names will be checked with the set 311 | # naming style. 312 | #variable-rgx= 313 | 314 | 315 | [CLASSES] 316 | 317 | # Warn about protected attribute access inside special methods 318 | check-protected-access-in-special-methods=no 319 | 320 | # List of method names used to declare (i.e. assign) instance attributes. 321 | defining-attr-methods=__init__, 322 | __new__, 323 | setUp, 324 | __post_init__ 325 | 326 | # List of member names, which should be excluded from the protected access 327 | # warning. 328 | exclude-protected=_asdict, 329 | _fields, 330 | _replace, 331 | _source, 332 | _make 333 | 334 | # List of valid names for the first argument in a class method. 335 | valid-classmethod-first-arg=cls 336 | 337 | # List of valid names for the first argument in a metaclass class method. 338 | valid-metaclass-classmethod-first-arg=cls 339 | 340 | 341 | [DESIGN] 342 | 343 | # List of regular expressions of class ancestor names to ignore when counting 344 | # public methods (see R0903) 345 | exclude-too-few-public-methods= 346 | 347 | # List of qualified class names to ignore when counting class parents (see 348 | # R0901) 349 | ignored-parents= 350 | 351 | # Maximum number of arguments for function / method. 352 | max-args=5 353 | 354 | # Maximum number of attributes for a class (see R0902). 355 | max-attributes=7 356 | 357 | # Maximum number of boolean expressions in an if statement (see R0916). 358 | max-bool-expr=5 359 | 360 | # Maximum number of branch for function / method body. 361 | max-branches=12 362 | 363 | # Maximum number of locals for function / method body. 364 | max-locals=15 365 | 366 | # Maximum number of parents for a class (see R0901). 367 | max-parents=7 368 | 369 | # Maximum number of public methods for a class (see R0904). 370 | max-public-methods=20 371 | 372 | # Maximum number of return / yield for function / method body. 373 | max-returns=6 374 | 375 | # Maximum number of statements in function / method body. 376 | max-statements=50 377 | 378 | # Minimum number of public methods for a class (see R0903). 379 | min-public-methods=2 380 | 381 | 382 | [EXCEPTIONS] 383 | 384 | # Exceptions that will emit a warning when caught. 385 | overgeneral-exceptions=BaseException, 386 | Exception 387 | 388 | 389 | [FORMAT] 390 | 391 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 392 | expected-line-ending-format= 393 | 394 | # Regexp for a line that is allowed to be longer than the limit. 395 | ignore-long-lines=^\s*(# )??$ 396 | 397 | # Number of spaces of indent required inside a hanging or continued line. 398 | indent-after-paren=4 399 | 400 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 401 | # tab). 402 | indent-string=' ' 403 | 404 | # Maximum number of characters on a single line. 405 | max-line-length=100 406 | 407 | # Maximum number of lines in a module. 408 | max-module-lines=1000 409 | 410 | # Allow the body of a class to be on the same line as the declaration if body 411 | # contains single statement. 412 | single-line-class-stmt=no 413 | 414 | # Allow the body of an if to be on the same line as the test if there is no 415 | # else. 416 | single-line-if-stmt=no 417 | 418 | 419 | [IMPORTS] 420 | 421 | # List of modules that can be imported at any level, not just the top level 422 | # one. 423 | allow-any-import-level= 424 | 425 | # Allow wildcard imports from modules that define __all__. 426 | allow-wildcard-with-all=no 427 | 428 | # Deprecated modules which should not be used, separated by a comma. 429 | deprecated-modules= 430 | 431 | # Output a graph (.gv or any supported image format) of external dependencies 432 | # to the given file (report RP0402 must not be disabled). 433 | ext-import-graph= 434 | 435 | # Output a graph (.gv or any supported image format) of all (i.e. internal and 436 | # external) dependencies to the given file (report RP0402 must not be 437 | # disabled). 438 | import-graph= 439 | 440 | # Output a graph (.gv or any supported image format) of internal dependencies 441 | # to the given file (report RP0402 must not be disabled). 442 | int-import-graph= 443 | 444 | # Force import order to recognize a module as part of the standard 445 | # compatibility libraries. 446 | known-standard-library= 447 | 448 | # Force import order to recognize a module as part of a third party library. 449 | known-third-party=enchant 450 | 451 | # Couples of modules and preferred modules, separated by a comma. 452 | preferred-modules= 453 | 454 | 455 | [LOGGING] 456 | 457 | # The type of string formatting that logging methods do. `old` means using % 458 | # formatting, `new` is for `{}` formatting. 459 | logging-format-style=old 460 | 461 | # Logging modules to check that the string format arguments are in logging 462 | # function parameter format. 463 | logging-modules=logging 464 | 465 | 466 | [MISCELLANEOUS] 467 | 468 | # List of note tags to take in consideration, separated by a comma. 469 | notes=FIXME, 470 | XXX, 471 | TODO 472 | 473 | # Regular expression of note tags to take in consideration. 474 | notes-rgx= 475 | 476 | 477 | [REFACTORING] 478 | 479 | # Maximum number of nested blocks for function / method body 480 | max-nested-blocks=5 481 | 482 | # Complete name of functions that never returns. When checking for 483 | # inconsistent-return-statements if a never returning function is called then 484 | # it will be considered as an explicit return statement and no message will be 485 | # printed. 486 | never-returning-functions=sys.exit,argparse.parse_error 487 | 488 | 489 | [SIMILARITIES] 490 | 491 | # Comments are removed from the similarity computation 492 | ignore-comments=yes 493 | 494 | # Docstrings are removed from the similarity computation 495 | ignore-docstrings=yes 496 | 497 | # Imports are removed from the similarity computation 498 | ignore-imports=yes 499 | 500 | # Signatures are removed from the similarity computation 501 | ignore-signatures=yes 502 | 503 | # Minimum lines number of a similarity. 504 | min-similarity-lines=4 505 | 506 | 507 | [SPELLING] 508 | 509 | # Limits count of emitted suggestions for spelling mistakes. 510 | max-spelling-suggestions=4 511 | 512 | # Spelling dictionary name. Available dictionaries: none. To make it work, 513 | # install the 'python-enchant' package. 514 | spelling-dict= 515 | 516 | # List of comma separated words that should be considered directives if they 517 | # appear at the beginning of a comment and should not be checked. 518 | spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: 519 | 520 | # List of comma separated words that should not be checked. 521 | spelling-ignore-words= 522 | 523 | # A path to a file that contains the private dictionary; one word per line. 524 | spelling-private-dict-file= 525 | 526 | # Tells whether to store unknown words to the private dictionary (see the 527 | # --spelling-private-dict-file option) instead of raising a message. 528 | spelling-store-unknown-words=no 529 | 530 | 531 | [STRING] 532 | 533 | # This flag controls whether inconsistent-quotes generates a warning when the 534 | # character used as a quote delimiter is used inconsistently within a module. 535 | check-quote-consistency=no 536 | 537 | # This flag controls whether the implicit-str-concat should generate a warning 538 | # on implicit string concatenation in sequences defined over several lines. 539 | check-str-concat-over-line-jumps=no 540 | 541 | 542 | [TYPECHECK] 543 | 544 | # List of decorators that produce context managers, such as 545 | # contextlib.contextmanager. Add to this list to register other decorators that 546 | # produce valid context managers. 547 | contextmanager-decorators=contextlib.contextmanager 548 | 549 | # List of members which are set dynamically and missed by pylint inference 550 | # system, and so shouldn't trigger E1101 when accessed. Python regular 551 | # expressions are accepted. 552 | generated-members= 553 | 554 | # Tells whether to warn about missing members when the owner of the attribute 555 | # is inferred to be None. 556 | ignore-none=yes 557 | 558 | # This flag controls whether pylint should warn about no-member and similar 559 | # checks whenever an opaque object is returned when inferring. The inference 560 | # can return multiple potential results while evaluating a Python object, but 561 | # some branches might not be evaluated, which results in partial inference. In 562 | # that case, it might be useful to still emit no-member and other checks for 563 | # the rest of the inferred objects. 564 | ignore-on-opaque-inference=yes 565 | 566 | # List of symbolic message names to ignore for Mixin members. 567 | ignored-checks-for-mixins=no-member, 568 | not-async-context-manager, 569 | not-context-manager, 570 | attribute-defined-outside-init 571 | 572 | # List of class names for which member attributes should not be checked (useful 573 | # for classes with dynamically set attributes). This supports the use of 574 | # qualified names. 575 | ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace 576 | 577 | # Show a hint with possible names when a member name was not found. The aspect 578 | # of finding the hint is based on edit distance. 579 | missing-member-hint=yes 580 | 581 | # The minimum edit distance a name should have in order to be considered a 582 | # similar match for a missing member name. 583 | missing-member-hint-distance=1 584 | 585 | # The total number of similar names that should be taken in consideration when 586 | # showing a hint for a missing member. 587 | missing-member-max-choices=1 588 | 589 | # Regex pattern to define which classes are considered mixins. 590 | mixin-class-rgx=.*[Mm]ixin 591 | 592 | # List of decorators that change the signature of a decorated function. 593 | signature-mutators= 594 | 595 | 596 | [VARIABLES] 597 | 598 | # List of additional names supposed to be defined in builtins. Remember that 599 | # you should avoid defining new builtins when possible. 600 | additional-builtins= 601 | 602 | # Tells whether unused global variables should be treated as a violation. 603 | allow-global-unused-variables=yes 604 | 605 | # List of names allowed to shadow builtins 606 | allowed-redefined-builtins= 607 | 608 | # List of strings which can identify a callback function by name. A callback 609 | # name must start or end with one of those strings. 610 | callbacks=cb_, 611 | _cb 612 | 613 | # A regular expression matching the name of dummy variables (i.e. expected to 614 | # not be used). 615 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 616 | 617 | # Argument names that match this expression will be ignored. Default to name 618 | # with leading underscore. 619 | ignored-argument-names=_.*|^ignored_|^unused_ 620 | 621 | # Tells whether we should check for unused import in __init__ files. 622 | init-import=no 623 | 624 | # List of qualified module names which can have objects that can redefine 625 | # builtins. 626 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 627 | --------------------------------------------------------------------------------