├── .codecov.yml ├── .coveragerc ├── .editorconfig ├── .github ├── FUNDING.yml └── workflows │ ├── ci.yml │ └── pypi_upload.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.rst ├── CODE_OF_CONDUCT.md ├── LICENSE ├── MANIFEST.in ├── Pipfile ├── Pipfile.lock ├── README.rst ├── docs ├── Makefile ├── conf.py ├── index.rst ├── make.bat ├── quickstart.rst ├── requirements.txt ├── vistir.cmdparse.rst ├── vistir.compat.rst ├── vistir.contextmanagers.rst ├── vistir.cursor.rst ├── vistir.misc.rst ├── vistir.path.rst ├── vistir.rst ├── vistir.spin.rst └── vistir.termcolors.rst ├── news └── .gitignore ├── pyproject.toml ├── setup.cfg ├── setup.py ├── src └── vistir │ ├── __init__.py │ ├── _winconsole.py │ ├── cmdparse.py │ ├── contextmanagers.py │ ├── cursor.py │ ├── misc.py │ ├── path.py │ ├── spin.py │ └── termcolors.py ├── tasks ├── CHANGELOG.rst.jinja2 └── __init__.py ├── tests ├── __init__.py ├── conftest.py ├── fixtures │ └── gutenberg_document.txt ├── strategies.py ├── test_contextmanagers.py ├── test_misc.py ├── test_path.py └── utils.py └── tox.ini /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | branch: master 3 | 4 | comment: off 5 | 6 | coverage: 7 | precision: 2 8 | round: nearest 9 | range: "50...100" 10 | 11 | status: 12 | project: yes 13 | patch: 14 | default: 15 | target: '50' 16 | changes: no 17 | 18 | ignore: 19 | - ".github" 20 | - "docs" 21 | - "tests" 22 | - "tasks" 23 | - "news" 24 | - "*.yml" 25 | - "*.ini" 26 | - "*.md" 27 | - "*.rst" 28 | - "*.txt" 29 | - "MANIFEST.in" 30 | - "*.cfg" 31 | - "*.toml" 32 | - "Pipfile" 33 | - "LICENSE" 34 | - "Pipfile.lock" 35 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = src/vistir/ 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | if self\.debug 14 | 15 | # Don't complain if tests don't hit defensive assertion code: 16 | raise AssertionError 17 | raise NotImplementedError 18 | # Don't complain if non-runnable code isn't run: 19 | if 0: 20 | if __name__ == .__main__.: 21 | 22 | # Don't complain about mypy hiding tools 23 | if MYPY_RUNNING: 24 | omit = 25 | src/vistir/_winconsole.py 26 | src/vistir/termcolors.py 27 | src/vistir/backports/* 28 | 29 | [html] 30 | directory = htmlcov 31 | 32 | [xml] 33 | output = coverage.xml 34 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | 14 | [*.toml] 15 | indent_size = 2 16 | 17 | [*.yaml] 18 | indent_size = 2 19 | 20 | # Makefiles always use tabs for indentation 21 | [Makefile] 22 | indent_style = tab 23 | 24 | # Batch files use tabs for indentation 25 | [*.bat] 26 | indent_style = tab 27 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [oz123,matteius] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | custom: # Replace with a single custom sponsorship URL 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Vistir CI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - 'news/**' 8 | - '*.ini' 9 | - '*.md' 10 | - '**/*.txt' 11 | - '*.rst' 12 | - '.gitignore' 13 | - '.gitmodules' 14 | - '.gitattributes' 15 | - '.editorconfig' 16 | branches: 17 | - master 18 | pull_request: 19 | paths-ignore: 20 | - 'docs/**' 21 | - 'news/**' 22 | - '*.ini' 23 | - '*.md' 24 | - '**/*.txt' 25 | - '*.rst' 26 | - '.gitignore' 27 | - '.gitmodules' 28 | - '.gitattributes' 29 | - '.editorconfig' 30 | 31 | jobs: 32 | test: 33 | name: ${{matrix.os}} / ${{ matrix.python-version }} 34 | runs-on: ${{ matrix.os }}-latest 35 | strategy: 36 | fail-fast: false 37 | matrix: 38 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 39 | os: [MacOS, Ubuntu, Windows] 40 | 41 | steps: 42 | - uses: actions/checkout@v2 43 | 44 | - name: Set up Python ${{ matrix.python-version }} 45 | uses: actions/setup-python@v2 46 | with: 47 | python-version: ${{ matrix.python-version }} 48 | 49 | - name: Get python path 50 | id: python-path 51 | run: | 52 | echo ::set-output name=path::$(python -c "import sys; print(sys.executable)") 53 | 54 | - name: Install latest pip, setuptools, wheel 55 | run: | 56 | python -m pip install --upgrade pip setuptools wheel --upgrade-strategy=eager 57 | python -m pip install --upgrade pipenv 58 | - name: Run tests 59 | env: 60 | PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} 61 | PYTHONWARNINGS: ignore:DEPRECATION 62 | PIPENV_NOSPIN: '1' 63 | CI: '1' 64 | COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} 65 | COVERALLS_GIT_COMMIT: ${{ github.sha }} 66 | COVERALLS_GIT_BRANCH: ${{ github.ref }} 67 | COVERALLS_FLAG_NAME: ${{ matrix.os }}-${{ matrix.python-version }} 68 | COVERALLS_SERVICE_NAME: github 69 | COVERALLS_SERVICE_JOB_ID: ${{ github.run_id }} 70 | COVERALLS_PARALLEL: true 71 | run: | 72 | git submodule sync 73 | git submodule update --init --recursive 74 | python -m pip install pipenv --upgrade 75 | python -m pipenv install --dev --python=${{ steps.python-path.outputs.path }} 76 | python -m pipenv run coverage run --concurrency=thread -m pytest -p no:xdist -ra tests/ 77 | -------------------------------------------------------------------------------- /.github/workflows/pypi_upload.yml: -------------------------------------------------------------------------------- 1 | name: Create Release & Upload To PyPI 2 | 3 | on: 4 | push: 5 | # Sequence of patterns matched against refs/tags 6 | tags: 7 | - v?[0-9]+.[0-9]+.[0-9]+ # add .* to allow dev releases 8 | 9 | jobs: 10 | build: 11 | name: vistir PyPI Upload 12 | runs-on: ubuntu-latest 13 | env: 14 | CI: "1" 15 | 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v2 19 | 20 | - name: Create Release 21 | id: create_release 22 | uses: actions/create-release@v1.0.0 23 | env: 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | with: 26 | tag_name: ${{ github.ref }} 27 | release_name: Release ${{ github.ref }} 28 | draft: false 29 | prerelease: false 30 | 31 | - name: Set up Python 3.7 32 | uses: actions/setup-python@v2 33 | with: 34 | python-version: 3.7 35 | 36 | - name: Install latest tools for build 37 | run: | 38 | python -m pip install --upgrade pip setuptools wheel pipenv invoke 39 | python -m pipenv install --dev --pre 40 | python -m pipenv run pip install invoke 41 | - name: Build wheels 42 | run: | 43 | python -m pipenv run inv build 44 | - name: Upload to PyPI via Twine 45 | # to upload to test pypi, pass --repository-url https://test.pypi.org/legacy/ and use secrets.TEST_PYPI_TOKEN 46 | run: | 47 | python -m pipenv run twine upload --verbose -u '__token__' -p '${{ secrets.PYPI_TOKEN }}' dist/* 48 | # git push https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:master 49 | # we need to use a deploy key for this to get around branch protection as the default token fails 50 | - name: Pre-bump 51 | run: | 52 | git config --local user.name 'Github Action' 53 | git config --local user.email action@github.com 54 | python -m pipenv run inv bump-version -t dev 55 | git commit -am 'pre-bump' 56 | 57 | - name: Push changes 58 | uses: ad-m/github-push-action@master 59 | with: 60 | github_token: ${{ secrets.GITHUB_TOKEN }} 61 | branch: master 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | pip-wheel-metadata/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | junit/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # javascript 62 | node_modules/ 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # Environments 90 | .env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | .vscode 111 | 112 | # pycharm 113 | .idea 114 | 115 | # Random caches 116 | pip-wheel-metadata/ 117 | XDG_CACHE_HOME/ 118 | 119 | *.sw[o,p] 120 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 22.6.0 4 | hooks: 5 | - id: black 6 | language_version: python3.7 7 | 8 | - repo: https://gitlab.com/pycqa/flake8 9 | rev: 3.8.1 # pick a git hash / tag to point to 10 | hooks: 11 | - id: flake8 12 | additional_dependencies: [flake8-bugbear] 13 | 14 | - repo: https://github.com/asottile/seed-isort-config 15 | rev: v2.1.1 16 | hooks: 17 | - id: seed-isort-config 18 | 19 | - repo: https://github.com/timothycrosley/isort 20 | rev: 4.3.21 21 | hooks: 22 | - id: isort 23 | 24 | - repo: https://github.com/asottile/pyupgrade 25 | rev: v2.4.1 26 | hooks: 27 | - id: pyupgrade 28 | 29 | - repo: https://github.com/myint/docformatter 30 | rev: v1.3.1 31 | hooks: 32 | - id: docformatter 33 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | 0.8.0 (2023-03-03) 2 | ================== 3 | 4 | * Drop spinner 5 | * Drop deprecated functions 6 | * Drop ability to import all from the package, 7 | instead user from ``vistir.``. 8 | 9 | 0.7.0 (2022-10-11) 10 | ================== 11 | 12 | Drop backports modules ``vistir.compat`` 13 | 14 | 0.6.1 (2022-07-27) 15 | ================== 16 | 17 | Bug fix 18 | 19 | 0.6.0 (2022-07-27) 20 | ================== 21 | 22 | Remove Python2 support 23 | 24 | 0.5.6 (2022-07-27) 25 | ================== 26 | 27 | No significant changes. 28 | 29 | 30 | 0.5.5 (2022-07-27) 31 | ================== 32 | 33 | No significant changes. 34 | 35 | 36 | 0.5.4 (2022-07-27) 37 | ================== 38 | 39 | No significant changes. 40 | 41 | 42 | 0.5.3 (2022-07-27) 43 | ================== 44 | 45 | Bug Fixes 46 | --------- 47 | 48 | - Fix bug where ``rmtree`` fails in non ``utf-8`` system. `#116 `_ 49 | 50 | - * Remove unsupported test runners for Python 2.7, 3.5 and 3.6 51 | * Add test runner for Python 3.9 52 | * remove reference to distutils 53 | * Update ``vistir`` Pipfile.lock after pinning ``pytest`` to prior working version 54 | * Remove ``pytype`` from the lock file as it was faiing to install 55 | * Remove ``coveralls`` from the CI 56 | * Remove Azure pipelines `#130 `_ 57 | 58 | 59 | 0.5.2 (2020-05-20) 60 | ================== 61 | 62 | Features 63 | -------- 64 | 65 | - ``vistir.compat`` now includes a backport of ``os.path.samefile`` for use on Windows on python 2.7. `#112 `_ 66 | 67 | 68 | 0.5.1 (2020-05-14) 69 | ================== 70 | 71 | Bug Fixes 72 | --------- 73 | 74 | - Fixed an issue which caused failures when calling ``contextmanagers.atomic_open_for_write`` due to assumed permission to call ``chmod`` which may not always be possible. `#110 `_ 75 | 76 | - Fixed several bugs with encoding of stream output on Windows and filesystem paths on OSX as well as Windows. `#111 `_ 77 | 78 | 79 | 0.5.0 (2020-01-13) 80 | ================== 81 | 82 | Features 83 | -------- 84 | 85 | - Reimplemented ``vistir.contextmanagers.open_file`` to fall back to ``urllib.urlopen`` in the absence of ``requests``, which is now an optional extra. `#102 `_ 86 | 87 | 88 | Bug Fixes 89 | --------- 90 | 91 | - Fixed a bug which caused ``path_to_url`` to sometimes fail to properly encode surrogates using utf-8 on windows using python 3. `#100 `_ 92 | 93 | 94 | 0.4.3 (2019-07-09) 95 | ================== 96 | 97 | Bug Fixes 98 | --------- 99 | 100 | - Added compatibility shim for ``TimeoutError`` exception handling. `#92 `_ 101 | 102 | - Exceptions are no longer suppressed after being handled during ``vistir.misc.run``. `#95 `_ 103 | 104 | - The signal handler for ``VistirSpinner`` will no longer cause deadlocks when ``CTRL_BREAK_EVENTS`` occur on windows. `#96 `_ 105 | 106 | 107 | 0.4.2 (2019-05-19) 108 | ================== 109 | 110 | Features 111 | -------- 112 | 113 | - Shortened windows paths will now be properly resolved to the full path by ``vistir.path.normalize_path``. `#90 `_ 114 | 115 | 116 | Bug Fixes 117 | --------- 118 | 119 | - Corrected argument order of ``icacls`` command for fixing permission issues when removing paths on windows. `#86 `_ 120 | 121 | - Fixed an issue which caused color wrapping of standard streams on windows to fail to surface critical attributes. `#88 `_ 122 | 123 | 124 | 0.4.1 (2019-05-15) 125 | ================== 126 | 127 | Features 128 | -------- 129 | 130 | - Added expanded functionality to assist with the removal of read-only paths on Windows via ``icacls`` system calls if necessary. `#81 `_ 131 | 132 | - Improved ``fs_encode`` compatibility shim in ``vistir.compat`` for handling of non-UTF8 data. `#83 `_ 133 | 134 | 135 | Bug Fixes 136 | --------- 137 | 138 | - Fixed a bug with ``vistir.misc.echo`` accidentally wrapping streams with ``colorama`` when it was not needed. 139 | Fixed a bug with rendering colors in text streams. `#82 `_ 140 | 141 | - Fixed ``vistir.misc.to_bytes`` implementation to respect supplied encoding. `#83 `_ 142 | 143 | - Blocking calls to ``vistir.misc.run`` will now properly handle ``KeyboardInterrupt`` events by terminating the subprocess and returning the result. `#84 `_ 144 | 145 | 146 | 0.4.0 (2019-04-10) 147 | ================== 148 | 149 | Features 150 | -------- 151 | 152 | - Added full native support for windows unicode consoles and the extended unicode character set when using ``vistir.misc.StreamWrapper`` instances via ``vistir.misc.get_wrapped_stream`` and ``vistir.misc.get_text_stream``. `#79 `_ 153 | 154 | 155 | Bug Fixes 156 | --------- 157 | 158 | - Fixed a bug which caused test failures due to generated paths on *nix based operating systems which were too long. `#65 `_ 159 | 160 | - Fixed a bug which caused spinner output to sometimes attempt to double encode on python 2, resulting in failed output encoding. `#69 `_ 161 | 162 | - Fixed a bug with the ``rmtree`` error handler implementation in ``compat.TemporaryDirectory`` which caused cleanup to fail intermittently on windows. `#72 `_ 163 | 164 | - Fixed an issue where paths could sometimes fail to be fs-encoded properly when using backported ``NamedTemporaryFile`` instances. `#74 `_ 165 | 166 | - Fixed a bug in ``vistir.misc.locale_encoding`` which caused invocation of a non-existent method called ``getlocaleencoding`` which forced all systems to use default encoding of ``ascii``. `#78 `_ 167 | 168 | 169 | 0.3.1 (2019-03-02) 170 | ================== 171 | 172 | Features 173 | -------- 174 | 175 | - Added a custom cursor hiding implementation to avoid depending on the cursor library, which was re-released under the GPL. `#57 `_ 176 | 177 | 178 | 0.3.0 (2019-01-01) 179 | ================== 180 | 181 | Features 182 | -------- 183 | 184 | - Added a new ``vistir.misc.StreamWrapper`` class with ``vistir.misc.get_wrapped_stream()`` to wrap existing streams 185 | and ``vistir.contextmanagers.replaced_stream()`` to temporarily replace a stream. `#48 `_ 186 | 187 | - Added new entries in ``vistir.compat`` to support movements to ``collections.abc``: ``Mapping``, ``Sequence``, ``Set``, ``ItemsView``. `#51 `_ 188 | 189 | - Improved ``decode_for_output`` to handle decoding failures gracefully by moving to an ``replace`` strategy. 190 | Now also allows a translation map to be provided to translate specific non-ascii characters when writing to outputs. `#52 `_ 191 | 192 | - Added support for properly encoding and decoding filesystem paths at the boundaries across python versions and platforms. `#53 `_ 193 | 194 | 195 | Bug Fixes 196 | --------- 197 | 198 | - Fix bug where FileNotFoundError is not imported from compat for rmtree `#46 `_ 199 | 200 | - Fixed a bug with exception handling during ``_create_process`` calls. `#49 `_ 201 | 202 | - Environment variables will now be properly passed through to ``run``. `#55 `_ 203 | 204 | 205 | 0.2.5 (2018-11-21) 206 | ================== 207 | 208 | Features 209 | -------- 210 | 211 | - Added the ability to always write spinner output to stderr using ``write_to_stdout=False``. `#40 `_ 212 | 213 | - Added extra path normalization and comparison utilities. `#42 `_ 214 | 215 | 216 | 0.2.4 (2018-11-12) 217 | ================== 218 | 219 | Features 220 | -------- 221 | 222 | - Remove additional text for ok and fail state `#35 `_ 223 | 224 | - Backported compatibility shims from ``CPython`` for improved cleanup of readonly temporary directories on cleanup. `#38 `_ 225 | 226 | 227 | 0.2.3 (2018-10-29) 228 | ================== 229 | 230 | Bug Fixes 231 | --------- 232 | 233 | - Improved handling of readonly path write-bit-setting. `#32 `_ 234 | 235 | - Fixed a bug with encoding of output streams for dummy spinner and formatting exceptions. `#33 `_ 236 | 237 | 238 | 0.2.2 (2018-10-26) 239 | ================== 240 | 241 | Bug Fixes 242 | --------- 243 | 244 | - Fixed a bug in the spinner implementation resulting in a failure to initialize colorama which could print control characters to the terminal on windows. `#30 `_ 245 | 246 | 247 | 0.2.1 (2018-10-26) 248 | ================== 249 | 250 | Features 251 | -------- 252 | 253 | - Implemented ``vistir.misc.create_tracked_tempdir``, which allows for automatically cleaning up resources using weakreferences at interpreter exit. `#26 `_ 254 | 255 | 256 | Bug Fixes 257 | --------- 258 | 259 | - Fixed a bug with string encodings for terminal colors when using spinners. `#27 `_ 260 | 261 | - Modified spinners to prefer to write to ``sys.stderr`` by default and to avoid writing ``None``, fixed an issue with signal registration on Windows. `#28 `_ 262 | 263 | 264 | 0.2.0 (2018-10-24) 265 | ================== 266 | 267 | Features 268 | -------- 269 | 270 | - Add windows compatible term colors and cursor toggles via custom spinner wrapper. `#19 `_ 271 | 272 | - Added new and improved functionality with fully integrated support for windows async non-unicode spinner. `#20 `_ 273 | 274 | - ``vistir.contextmanager.spinner`` and ``vistir.spin.VistirSpinner`` now provide ``write_err`` to write to standard error from the spinner. `#22 `_ 275 | 276 | - Added ``vistir.path.create_tracked_tempfile`` to the API for weakref-tracked temporary files. `#26 `_ 277 | 278 | 279 | Bug Fixes 280 | --------- 281 | 282 | - Add compatibility shim for ``WindowsError`` issues. `#18 `_ 283 | 284 | - ``vistir.contextmanager.spinner`` and ``vistir.spin.VistirSpinner`` now provide ``write_err`` to write to standard error from the spinner. `#23 `_ 285 | 286 | - Suppress ``ResourceWarning`` at runtime if warnings are suppressed in general. `#24 `_ 287 | 288 | 289 | 0.1.7 (2018-10-11) 290 | ================== 291 | 292 | Features 293 | -------- 294 | 295 | - Updated ``misc.run`` to accept new arguments for ``spinner``, ``combine_stderr``, and ``display_limit``. `#16 `_ 296 | 297 | 298 | 0.1.6 (2018-09-13) 299 | ================== 300 | 301 | Features 302 | -------- 303 | 304 | - Made ``yaspin`` an optional dependency which can be added as an extra by using ``pip install vistir[spinner]`` and can be toggled with ``vistir.misc.run(...nospin=True)``. `#12 `_ 305 | 306 | - Added ``verbose`` flag to ``vistir.misc.run()`` to provide a way to prevent printing all subprocess output. `#13 `_ 307 | 308 | 309 | 0.1.5 (2018-09-07) 310 | ================== 311 | 312 | Features 313 | -------- 314 | 315 | - Users may now pass ``block=False`` to create nonblocking subprocess calls to ``vistir.misc.run()``. 316 | ``vistir.misc.run()`` will now provide a spinner when passed ``spinner=True``. `#11 `_ 317 | 318 | 319 | Bug Fixes 320 | --------- 321 | 322 | - ``vistir.misc.run()`` now provides the full subprocess object without communicating with it when passed ``return_object=True``. `#11 `_ 323 | 324 | 325 | 0.1.4 (2018-08-18) 326 | ================== 327 | 328 | Features 329 | -------- 330 | 331 | - Implemented ``vistir.path.ensure_mkdir_p`` decorator for wrapping the output of a function call to ensure it is created as a directory. 332 | 333 | Added ``vistir.path.create_tracked_tmpdir`` functionality for creating a temporary directory which is tracked using an ``atexit`` handler rather than a context manager. `#7 `_ 334 | 335 | 336 | Bug Fixes 337 | --------- 338 | 339 | - Use native implementation of ``os.makedirs`` to fix still-broken ``mkdir_p`` but provide additional error-handling logic. `#6 `_ 340 | 341 | 342 | 0.1.3 (2018-08-18) 343 | ================== 344 | 345 | Bug Fixes 346 | --------- 347 | 348 | - Fixed an issue which caused ``mkdir_p`` to use incorrect permissions and throw errors when creating intermediary paths. `#6 `_ 349 | 350 | 351 | 0.1.2 (2018-08-18) 352 | ================== 353 | 354 | Features 355 | -------- 356 | 357 | - Added ``mode`` parameter to ``vistir.path.mkdir_p``. `#5 `_ 358 | 359 | 360 | 0.1.1 (2018-08-14) 361 | ================== 362 | 363 | Features 364 | -------- 365 | 366 | - Added support for coverage and tox builds. `#2 `_ 367 | 368 | - Enhanced subprocess runner to reproduce the behavior of pipenv's subprocess runner. `#4 `_ 369 | 370 | 371 | Bug Fixes 372 | --------- 373 | 374 | - Fixed an issue where ``vistir.misc.run`` would fail to encode environment variables to the proper filesystem encoding on windows. `#1 `_ 375 | 376 | - Fixed encoding issues when passing commands and environments to ``vistir.misc.run()``. `#3 `_ 377 | 378 | 379 | 0.1.0 (2018-08-12) 380 | ======================= 381 | 382 | Features 383 | -------- 384 | 385 | - Initial commit and release `#0 `_ 386 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at dan@danryan.co. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, Dan Ryan 2 | 3 | Permission to use, copy, modify, and distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE* README* 2 | include CHANGELOG.rst 3 | include CODE_OF_CONDUCT.md 4 | include pyproject.toml 5 | 6 | exclude .editorconfig 7 | exclude .coveragerc 8 | exclude .pre-commit-config.yaml 9 | exclude .travis.yml 10 | exclude azure-pipelines.yml 11 | exclude tox.ini 12 | exclude appveyor.yml 13 | exclude Pipfile* 14 | 15 | recursive-include docs Makefile *.rst *.py *.bat 16 | recursive-exclude docs requirements*.txt 17 | 18 | prune .github 19 | prune docs/build 20 | prune news 21 | prune tasks 22 | prune tests 23 | prune .azure-pipelines 24 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [packages] 2 | vistir = {editable = true, extras = ["requests", "spinner", "tests", "dev", "typing"], path = "."} 3 | 4 | [dev-packages] 5 | towncrier = '*' 6 | wheel = '*' 7 | codacy-coverage = "*" 8 | codecov = "*" 9 | 10 | [scripts] 11 | release = 'inv release' 12 | black = 'black src/vistir/ --exclude "/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist|src/vistir/_vendor)/"' 13 | tests = "pytest -v tests" 14 | draft = "towncrier --draft" 15 | changelog = "towncrier" 16 | build = "setup.py sdist bdist_wheel" 17 | upload = "twine upload dist/*" 18 | docs = "inv build-docs" 19 | mdchangelog = "pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md" 20 | 21 | [pipenv] 22 | allow_prereleases = true 23 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================================================================== 2 | vistir: Setup / utilities which most projects eventually need 3 | =============================================================================== 4 | 5 | .. image:: https://img.shields.io/pypi/v/vistir.svg 6 | :target: https://pypi.python.org/pypi/vistir 7 | 8 | .. image:: https://img.shields.io/pypi/l/vistir.svg 9 | :target: https://pypi.python.org/pypi/vistir 10 | 11 | .. image:: https://travis-ci.com/sarugaku/vistir.svg?branch=master 12 | :target: https://travis-ci.com/sarugaku/vistir 13 | 14 | .. image:: https://img.shields.io/pypi/pyversions/vistir.svg 15 | :target: https://pypi.python.org/pypi/vistir 16 | 17 | .. image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg 18 | :target: https://saythanks.io/to/techalchemy 19 | 20 | .. image:: https://readthedocs.org/projects/vistir/badge/?version=latest 21 | :target: https://vistir.readthedocs.io/en/latest/?badge=latest 22 | :alt: Documentation Status 23 | 24 | .. image:: https://dev.azure.com/sarugaku/vistir/_apis/build/status/Vistir%20Build%20Pipeline?branchName=master 25 | :target: https://dev.azure.com/sarugaku/vistir/_build/latest?definitionId=2&branchName=master 26 | 27 | 28 | 🐉 Installation 29 | ================= 30 | 31 | Install from `PyPI`_: 32 | 33 | :: 34 | 35 | $ pipenv install vistir 36 | 37 | Install from `Github`_: 38 | 39 | :: 40 | 41 | $ pipenv install -e git+https://github.com/sarugaku/vistir.git#egg=vistir 42 | 43 | 44 | .. _PyPI: https://www.pypi.org/project/vistir 45 | .. _Github: https://github.com/sarugaku/vistir 46 | 47 | 48 | .. _`Summary`: 49 | 50 | 🐉 Summary 51 | =========== 52 | 53 | **vistir** is a library full of utility functions designed to make life easier. Here are 54 | some of the places where these functions are used: 55 | 56 | * `pipenv`_ 57 | * `requirementslib`_ 58 | * `pip-tools`_ 59 | * `passa`_ 60 | * `pythonfinder`_ 61 | 62 | .. _passa: https://github.com/sarugaku/passa 63 | .. _pipenv: https://github.com/pypa/pipenv 64 | .. _pip-tools: https://github.com/jazzband/pip-tools 65 | .. _requirementslib: https://github.com/sarugaku/requirementslib 66 | .. _pythonfinder: https://github.com/sarugaku/pythonfinder 67 | 68 | 69 | .. _`Usage`: 70 | 71 | 🐉 Usage 72 | ========== 73 | 74 | Importing a utility 75 | -------------------- 76 | 77 | You can import utilities directly from **vistir**: 78 | 79 | .. code:: python 80 | 81 | from vistir import cd 82 | with cd('/path/to/somedir'): 83 | do_stuff_in('somedir') 84 | 85 | 86 | .. _`Functionality`: 87 | 88 | 🐉 Functionality 89 | ================== 90 | 91 | **vistir** provides several categories of functionality, including: 92 | 93 | * Compatibility Shims 94 | * Context Managers 95 | * Miscellaneous Utilities 96 | * Path Utilities 97 | 98 | 🐉 Context Managers 99 | -------------------- 100 | 101 | **vistir** provides the following context managers as utility contexts: 102 | 103 | * ``vistir.contextmanagers.atomic_open_for_write`` 104 | * ``vistir.contextmanagers.cd`` 105 | * ``vistir.contextmanagers.open_file`` 106 | * ``vistir.contextmanagers.replaced_stream`` 107 | * ``vistir.contextmanagers.replaced_streams`` 108 | * ``vistir.contextmanagers.temp_environ`` 109 | * ``vistir.contextmanagers.temp_path`` 110 | 111 | 112 | .. _`atomic_open_for_write`: 113 | 114 | **atomic_open_for_write** 115 | /////////////////////////// 116 | 117 | This context manager ensures that a file only gets overwritten if the contents can be 118 | successfully written in its place. If you open a file for writing and then fail in the 119 | middle under normal circumstances, your original file is already gone. 120 | 121 | .. code:: python 122 | 123 | >>> fn = "test_file.txt" 124 | >>> with open(fn, "w") as fh: 125 | fh.write("this is some test text") 126 | >>> read_test_file() 127 | this is some test text 128 | >>> def raise_exception_while_writing(filename): 129 | with vistir.contextmanagers.atomic_open_for_write(filename) as fh: 130 | fh.write("Overwriting all the text from before with even newer text") 131 | raise RuntimeError("But did it get overwritten now?") 132 | >>> raise_exception_while_writing(fn) 133 | Traceback (most recent call last): 134 | ... 135 | RuntimeError: But did it get overwritten now? 136 | >>> read_test_file() 137 | this is some test text 138 | 139 | 140 | .. _`cd`: 141 | 142 | **cd** 143 | /////// 144 | 145 | A context manager for temporarily changing the working directory. 146 | 147 | 148 | .. code:: python 149 | 150 | >>> os.path.abspath(os.curdir) 151 | '/tmp/test' 152 | >>> with vistir.contextmanagers.cd('/tmp/vistir_test'): 153 | print(os.path.abspath(os.curdir)) 154 | /tmp/vistir_test 155 | 156 | 157 | .. _`open_file`: 158 | 159 | **open_file** 160 | /////////////// 161 | 162 | A context manager for streaming file contents, either local or remote. It is recommended 163 | to pair this with an iterator which employs a sensible chunk size. 164 | 165 | 166 | .. code:: python 167 | 168 | >>> filecontents = b"" 169 | with vistir.contextmanagers.open_file("https://norvig.com/big.txt") as fp: 170 | for chunk in iter(lambda: fp.read(16384), b""): 171 | filecontents.append(chunk) 172 | >>> import io 173 | >>> import shutil 174 | >>> filecontents = io.BytesIO(b"") 175 | >>> with vistir.contextmanagers.open_file("https://norvig.com/big.txt") as fp: 176 | shutil.copyfileobj(fp, filecontents) 177 | 178 | 179 | **replaced_stream** 180 | //////////////////// 181 | 182 | .. _`replaced_stream`: 183 | 184 | A context manager to temporarily swap out *stream_name* with a stream wrapper. This will 185 | capture the stream output and prevent it from being written as normal. 186 | 187 | .. code-block:: python 188 | 189 | >>> orig_stdout = sys.stdout 190 | >>> with replaced_stream("stdout") as stdout: 191 | ... sys.stdout.write("hello") 192 | ... assert stdout.getvalue() == "hello" 193 | ... assert orig_stdout.getvalue() != "hello" 194 | 195 | >>> sys.stdout.write("hello") 196 | 'hello' 197 | 198 | 199 | .. _`replaced_streams`: 200 | 201 | **replaced_streams** 202 | ///////////////////// 203 | 204 | 205 | Temporarily replaces both *sys.stdout* and *sys.stderr* and captures anything written 206 | to these respective targets. 207 | 208 | 209 | .. code-block:: python 210 | 211 | >>> import sys 212 | >>> with vistir.contextmanagers.replaced_streams() as streams: 213 | >>> stdout, stderr = streams 214 | >>> sys.stderr.write("test") 215 | >>> sys.stdout.write("hello") 216 | >>> assert stdout.getvalue() == "hello" 217 | >>> assert stderr.getvalue() == "test" 218 | 219 | >>> stdout.getvalue() 220 | 'hello' 221 | 222 | >>> stderr.getvalue() 223 | 'test' 224 | 225 | 226 | .. _`spinner`: 227 | 228 | **spinner** 229 | //////////// 230 | 231 | A context manager for wrapping some actions with a threaded, interrupt-safe spinner. The 232 | spinner is fully compatible with all terminals (you can use ``bouncingBar`` on non-utf8 233 | terminals) and will allow you to update the text of the spinner itself by simply setting 234 | ``spinner.text`` or write lines to the screen above the spinner by using 235 | ``spinner.write(line)``. Success text can be indicated using ``spinner.ok("Text")`` and 236 | failure text can be indicated with ``spinner.fail("Fail text")``. 237 | 238 | .. code:: python 239 | 240 | >>> lines = ["a", "b"] 241 | >>> with vistir.contextmanagers.spinner(spinner_name="dots", text="Running...", handler_map={}, nospin=False) as sp: 242 | for line in lines: 243 | sp.write(line + "\n") 244 | while some_variable = some_queue.pop(): 245 | sp.text = "Consuming item: %s" % some_variable 246 | if success_condition: 247 | sp.ok("Succeeded!") 248 | else: 249 | sp.fail("Failed!") 250 | 251 | 252 | .. _`temp_environ`: 253 | 254 | **temp_environ** 255 | ///////////////// 256 | 257 | Sets a temporary environment context to freely manipulate ``os.environ`` which will 258 | be reset upon exiting the context. 259 | 260 | 261 | .. code:: python 262 | 263 | >>> os.environ['MY_KEY'] = "test" 264 | >>> os.environ['MY_KEY'] 265 | 'test' 266 | >>> with vistir.contextmanagers.temp_environ(): 267 | os.environ['MY_KEY'] = "another thing" 268 | print("New key: %s" % os.environ['MY_KEY']) 269 | New key: another thing 270 | >>> os.environ['MY_KEY'] 271 | 'test' 272 | 273 | 274 | .. _`temp_path`: 275 | 276 | **temp_path** 277 | ////////////// 278 | 279 | Sets a temporary environment context to freely manipulate ``sys.path`` which will 280 | be reset upon exiting the context. 281 | 282 | 283 | .. code:: python 284 | 285 | >>> path_from_virtualenv = load_path("/path/to/venv/bin/python") 286 | >>> print(sys.path) 287 | ['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'] 288 | >>> with temp_path(): 289 | sys.path = path_from_virtualenv 290 | # Running in the context of the path above 291 | run(["pip", "install", "stuff"]) 292 | >>> print(sys.path) 293 | ['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'] 294 | 295 | 296 | 🐉 Cursor Utilities 297 | -------------------------- 298 | 299 | The following Cursor utilities are available to manipulate the console cursor: 300 | 301 | * ``vistir.cursor.hide_cursor`` 302 | * ``vistir.cursor.show_cursor`` 303 | 304 | 305 | .. _`hide_cursor`: 306 | 307 | **hide_cursor** 308 | ///////////////// 309 | 310 | Hide the console cursor in the given stream. 311 | 312 | .. code:: python 313 | 314 | >>> vistir.cursor.hide_cursor(stream=sys.stdout) 315 | 316 | 317 | .. _`show_cursor`: 318 | 319 | **show_cursor** 320 | ///////////////// 321 | 322 | Show the console cursor in the given stream. 323 | 324 | .. code:: python 325 | 326 | >>> vistir.cursor.show_cursor(stream=sys.stdout) 327 | 328 | 329 | 🐉 Miscellaneous Utilities 330 | -------------------------- 331 | 332 | The following Miscellaneous utilities are available as helper methods: 333 | 334 | * ``vistir.misc.shell_escape`` 335 | * ``vistir.misc.unnest`` 336 | * ``vistir.misc.run`` 337 | * ``vistir.misc.load_path`` 338 | * ``vistir.misc.partialclass`` 339 | * ``vistir.misc.to_text`` 340 | * ``vistir.misc.to_bytes`` 341 | * ``vistir.misc.take`` 342 | * ``vistir.misc.decode_for_output`` 343 | * ``vistir.misc.get_canonical_encoding_name`` 344 | * ``vistir.misc.get_wrapped_stream`` 345 | * ``vistir.misc.StreamWrapper`` 346 | * ``vistir.misc.get_text_stream`` 347 | * ``vistir.misc.replace_with_text_stream`` 348 | * ``vistir.misc.get_text_stdin`` 349 | * ``vistir.misc.get_text_stdout`` 350 | * ``vistir.misc.get_text_stderr`` 351 | * ``vistir.misc.echo`` 352 | 353 | 354 | .. _`shell_escape`: 355 | 356 | **shell_escape** 357 | ///////////////// 358 | 359 | Escapes a string for use as shell input when passing *shell=True* to ``os.Popen``. 360 | 361 | .. code:: python 362 | 363 | >>> vistir.misc.shell_escape("/tmp/test/test script.py hello") 364 | '/tmp/test/test script.py hello' 365 | 366 | 367 | .. _`unnest`: 368 | 369 | **unnest** 370 | /////////// 371 | 372 | Unnests nested iterables into a flattened one. 373 | 374 | .. code:: python 375 | 376 | >>> nested_iterable = (1234, (3456, 4398345, (234234)), (2396, (23895750, 9283798, 29384, (289375983275, 293759, 2347, (2098, 7987, 27599))))) 377 | >>> list(vistir.misc.unnest(nested_iterable)) 378 | [1234, 3456, 4398345, 234234, 2396, 23895750, 9283798, 29384, 289375983275, 293759, 2347, 2098, 7987, 27599] 379 | 380 | 381 | .. _`dedup`: 382 | 383 | **dedup** 384 | ////////// 385 | 386 | Deduplicates an iterable (like a ``set``, but preserving order). 387 | 388 | .. code:: python 389 | 390 | >>> iterable = ["repeatedval", "uniqueval", "repeatedval", "anotherval", "somethingelse"] 391 | >>> list(vistir.misc.dedup(iterable)) 392 | ['repeatedval', 'uniqueval', 'anotherval', 'somethingelse'] 393 | 394 | .. _`run`: 395 | 396 | **run** 397 | //////// 398 | 399 | Runs the given command using ``subprocess.Popen`` and passing sane defaults. 400 | 401 | .. code:: python 402 | 403 | >>> out, err = vistir.run(["cat", "/proc/version"]) 404 | >>> out 405 | 'Linux version 4.15.0-27-generic (buildd@lgw01-amd64-044) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #29-Ubuntu SMP Wed Jul 11 08:21:57 UTC 2018' 406 | 407 | 408 | .. _`load_path`: 409 | 410 | **load_path** 411 | ////////////// 412 | 413 | Load the ``sys.path`` from the given python executable's environment as json. 414 | 415 | .. code:: python 416 | 417 | >>> load_path("/home/user/.virtualenvs/requirementslib-5MhGuG3C/bin/python") 418 | ['', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python37.zip', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/site-packages', '/home/user/git/requirementslib/src'] 419 | 420 | 421 | .. _`partialclass`: 422 | 423 | **partialclass** 424 | ///////////////// 425 | 426 | Create a partially instantiated class. 427 | 428 | .. code:: python 429 | 430 | >>> source = partialclass(Source, url="https://pypi.org/simple") 431 | >>> new_source = source(name="pypi") 432 | >>> new_source 433 | <__main__.Source object at 0x7f23af189b38> 434 | >>> new_source.__dict__ 435 | {'url': 'https://pypi.org/simple', 'verify_ssl': True, 'name': 'pypi'} 436 | 437 | 438 | .. _`to_text`: 439 | 440 | **to_text** 441 | //////////// 442 | 443 | Convert arbitrary text-formattable input to text while handling errors. 444 | 445 | .. code:: python 446 | 447 | >>> vistir.misc.to_text(b"these are bytes") 448 | 'these are bytes' 449 | 450 | 451 | .. _`to_bytes`: 452 | 453 | **to_bytes** 454 | ///////////// 455 | 456 | Converts arbitrary byte-convertable input to bytes while handling errors. 457 | 458 | .. code:: python 459 | 460 | >>> vistir.misc.to_bytes("this is some text") 461 | b'this is some text' 462 | >>> vistir.misc.to_bytes(u"this is some text") 463 | b'this is some text' 464 | 465 | 466 | .. _`chunked`: 467 | 468 | **chunked** 469 | //////////// 470 | 471 | Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. 472 | 473 | This example will create groups of chunk size **5**, which means there will be *6 groups*. 474 | 475 | .. code-block:: python 476 | 477 | >>> chunked_iterable = vistir.misc.chunked(5, range(30)) 478 | >>> for chunk in chunked_iterable: 479 | ... add_to_some_queue(chunk) 480 | 481 | .. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping 482 | 483 | 484 | .. _`take`: 485 | 486 | **take** 487 | ///////// 488 | 489 | Take elements from the supplied iterable without consuming it. 490 | 491 | .. code-block:: python 492 | 493 | >>> iterable = range(30) 494 | >>> first_10 = take(10, iterable) 495 | >>> [i for i in first_10] 496 | [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] 497 | 498 | >>> [i for i in iterable] 499 | [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] 500 | 501 | 502 | .. _`divide`: 503 | 504 | **divide** 505 | //////////// 506 | 507 | Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. 508 | 509 | .. code-block:: python 510 | 511 | >>> iterable = range(30) 512 | >>> groups = [] 513 | >>> for grp in vistir.misc.divide(3, iterable): 514 | ... groups.append(grp) 515 | >>> groups 516 | [, , ] 517 | 518 | 519 | .. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping 520 | 521 | 522 | .. _`decode_for_output`: 523 | 524 | **decode_for_output** 525 | ////////////////////// 526 | 527 | Converts an arbitrary text input to output which is encoded for printing to terminal 528 | outputs using the system preferred locale using ``locale.getpreferredencoding(False)`` 529 | with some additional hackery on linux systems. 530 | 531 | 532 | .. _`get_canonical_encoding_name`: 533 | 534 | **get_canonical_encoding_name** 535 | //////////////////////////////// 536 | 537 | Given an encoding name, get the canonical name from a codec lookup. 538 | 539 | .. code-block:: python 540 | 541 | >>> vistir.misc.get_canonical_encoding_name("utf8") 542 | "utf-8" 543 | 544 | 545 | .. _`get_wrapped_stream`: 546 | 547 | **get_wrapped_stream** 548 | ////////////////////// 549 | 550 | Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. 551 | 552 | .. code-block:: python 553 | 554 | >>> stream = sys.stdout 555 | >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) 556 | >>> wrapped_stream.write("unicode\u0141") 557 | >>> wrapped_stream.seek(0) 558 | >>> wrapped_stream.read() 559 | "unicode\u0141" 560 | 561 | 562 | .. _`StreamWrapper`: 563 | 564 | **StreamWrapper** 565 | ////////////////// 566 | 567 | A stream wrapper and compatibility class for handling wrapping file-like stream objects 568 | which may be used in place of ``sys.stdout`` and other streams. 569 | 570 | .. code-block:: python 571 | 572 | >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) 573 | >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) 574 | 575 | 576 | .. _`get_text_stream`: 577 | 578 | **get_text_stream** 579 | //////////////////// 580 | 581 | An implementation of the **StreamWrapper** for the purpose of wrapping **sys.stdin** or **sys.stdout**. 582 | 583 | On Windows, this returns the appropriate handle to the requested output stream. 584 | 585 | .. code-block:: python 586 | 587 | >>> text_stream = vistir.misc.get_text_stream("stdout") 588 | >>> sys.stdout = text_stream 589 | >>> sys.stdin = vistir.misc.get_text_stream("stdin") 590 | >>> vistir.misc.echo(u"\0499", fg="green") 591 | ҙ 592 | 593 | 594 | .. _`replace_with_text_stream`: 595 | 596 | **replace_with_text_stream** 597 | ///////////////////////////// 598 | 599 | Given a text stream name, replaces the text stream with a **StreamWrapper** instance. 600 | 601 | 602 | .. code-block:: python 603 | 604 | >>> vistir.misc.replace_with_text_stream("stdout") 605 | 606 | Once invoked, the standard stream in question is replaced with the required wrapper, 607 | turning it into a ``TextIOWrapper`` compatible stream (which ensures that unicode 608 | characters can be written to it). 609 | 610 | 611 | .. _`get_text_stdin`: 612 | 613 | **get_text_stdin** 614 | /////////////////// 615 | 616 | A helper function for calling **get_text_stream("stdin")**. 617 | 618 | 619 | .. _`get_text_stdout`: 620 | 621 | **get_text_stdout** 622 | //////////////////// 623 | 624 | A helper function for calling **get_text_stream("stdout")**. 625 | 626 | 627 | .. _`get_text_stderr`: 628 | 629 | **get_text_stderr** 630 | //////////////////// 631 | 632 | A helper function for calling **get_text_stream("stderr")**. 633 | 634 | 635 | .. _`echo`: 636 | 637 | **echo** 638 | ///////// 639 | 640 | Writes colored, stream-compatible output to the desired handle (``sys.stdout`` by default). 641 | 642 | .. code-block:: python 643 | 644 | >>> vistir.misc.echo("some text", fg="green", bg="black", style="bold", err=True) # write to stderr 645 | some text 646 | >>> vistir.misc.echo("some other text", fg="cyan", bg="white", style="underline") # write to stdout 647 | some other text 648 | 649 | 650 | 🐉 Path Utilities 651 | ------------------ 652 | 653 | **vistir** provides utilities for interacting with filesystem paths: 654 | 655 | * ``vistir.path.get_converted_relative_path`` 656 | * ``vistir.path.normalize_path`` 657 | * ``vistir.path.is_in_path`` 658 | * ``vistir.path.handle_remove_readonly`` 659 | * ``vistir.path.is_file_url`` 660 | * ``vistir.path.is_readonly_path`` 661 | * ``vistir.path.is_valid_url`` 662 | * ``vistir.path.mkdir_p`` 663 | * ``vistir.path.ensure_mkdir_p`` 664 | * ``vistir.path.create_tracked_tempdir`` 665 | * ``vistir.path.create_tracked_tempfile`` 666 | * ``vistir.path.path_to_url`` 667 | * ``vistir.path.rmtree`` 668 | * ``vistir.path.safe_expandvars`` 669 | * ``vistir.path.set_write_bit`` 670 | * ``vistir.path.url_to_path`` 671 | * ``vistir.path.walk_up`` 672 | 673 | 674 | .. _`normalize_path`: 675 | 676 | **normalize_path** 677 | ////////////////// 678 | 679 | Return a case-normalized absolute variable-expanded path. 680 | 681 | 682 | .. code:: python 683 | 684 | >>> vistir.path.normalize_path("~/${USER}") 685 | /home/user/user 686 | 687 | 688 | .. _`is_in_path`: 689 | 690 | **is_in_path** 691 | ////////////// 692 | 693 | Determine if the provided full path is in the given parent root. 694 | 695 | 696 | .. code:: python 697 | 698 | >>> vistir.path.is_in_path("~/.pyenv/versions/3.7.1/bin/python", "${PYENV_ROOT}/versions") 699 | True 700 | 701 | 702 | .. _`get_converted_relative_path`: 703 | 704 | **get_converted_relative_path** 705 | //////////////////////////////// 706 | 707 | Convert the supplied path to a relative path (relative to ``os.curdir``) 708 | 709 | 710 | .. code:: python 711 | 712 | >>> os.chdir('/home/user/code/myrepo/myfolder') 713 | >>> vistir.path.get_converted_relative_path('/home/user/code/file.zip') 714 | './../../file.zip' 715 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder/mysubfolder') 716 | './mysubfolder' 717 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') 718 | '.' 719 | 720 | 721 | .. _`handle_remove_readonly`: 722 | 723 | **handle_remove_readonly** 724 | /////////////////////////// 725 | 726 | Error handler for shutil.rmtree. 727 | 728 | Windows source repo folders are read-only by default, so this error handler attempts to 729 | set them as writeable and then proceed with deletion. 730 | 731 | This function will call check ``vistir.path.is_readonly_path`` before attempting to 732 | call ``vistir.path.set_write_bit`` on the target path and try again. 733 | 734 | 735 | .. _`is_file_url`: 736 | 737 | **is_file_url** 738 | //////////////// 739 | 740 | Checks whether the given url is a properly formatted ``file://`` uri. 741 | 742 | .. code:: python 743 | 744 | >>> vistir.path.is_file_url('file:///home/user/somefile.zip') 745 | True 746 | >>> vistir.path.is_file_url('/home/user/somefile.zip') 747 | False 748 | 749 | 750 | .. _`is_readonly_path`: 751 | 752 | **is_readonly_path** 753 | ///////////////////// 754 | 755 | Check if a provided path exists and is readonly by checking for ``bool(path.stat & stat.S_IREAD) and not os.access(path, os.W_OK)`` 756 | 757 | .. code:: python 758 | 759 | >>> vistir.path.is_readonly_path('/etc/passwd') 760 | True 761 | >>> vistir.path.is_readonly_path('/home/user/.bashrc') 762 | False 763 | 764 | 765 | .. _`is_valid_url`: 766 | 767 | **is_valid_url** 768 | ///////////////// 769 | 770 | Checks whether a URL is valid and parseable by checking for the presence of a scheme and 771 | a netloc. 772 | 773 | .. code:: python 774 | 775 | >>> vistir.path.is_valid_url("https://google.com") 776 | True 777 | >>> vistir.path.is_valid_url("/home/user/somefile") 778 | False 779 | 780 | 781 | .. _`mkdir_p`: 782 | 783 | **mkdir_p** 784 | ///////////// 785 | 786 | Recursively creates the target directory and all of its parents if they do not 787 | already exist. Fails silently if they do. 788 | 789 | .. code:: python 790 | 791 | >>> os.mkdir('/tmp/test_dir') 792 | >>> os.listdir('/tmp/test_dir') 793 | [] 794 | >>> vistir.path.mkdir_p('/tmp/test_dir/child/subchild/subsubchild') 795 | >>> os.listdir('/tmp/test_dir/child/subchild') 796 | ['subsubchild'] 797 | 798 | 799 | .. _`ensure_mkdir_p`: 800 | 801 | **ensure_mkdir_p** 802 | /////////////////// 803 | 804 | A decorator which ensures that the caller function's return value is created as a 805 | directory on the filesystem. 806 | 807 | .. code:: python 808 | 809 | >>> @ensure_mkdir_p 810 | def return_fake_value(path): 811 | return path 812 | >>> return_fake_value('/tmp/test_dir') 813 | >>> os.listdir('/tmp/test_dir') 814 | [] 815 | >>> return_fake_value('/tmp/test_dir/child/subchild/subsubchild') 816 | >>> os.listdir('/tmp/test_dir/child/subchild') 817 | ['subsubchild'] 818 | 819 | 820 | .. _`create_tracked_tempdir`: 821 | 822 | **create_tracked_tempdir** 823 | //////////////////////////// 824 | 825 | Creates a tracked temporary directory using ``vistir.path.TemporaryDirectory``, but does 826 | not remove the directory when the return value goes out of scope, instead registers a 827 | handler to cleanup on program exit. 828 | 829 | .. code:: python 830 | 831 | >>> temp_dir = vistir.path.create_tracked_tempdir(prefix="test_dir") 832 | >>> assert temp_dir.startswith("test_dir") 833 | True 834 | >>> with vistir.path.create_tracked_tempdir(prefix="test_dir") as temp_dir: 835 | with io.open(os.path.join(temp_dir, "test_file.txt"), "w") as fh: 836 | fh.write("this is a test") 837 | >>> os.listdir(temp_dir) 838 | 839 | 840 | .. _`create_tracked_tempfile`: 841 | 842 | **create_tracked_tempfile** 843 | //////////////////////////// 844 | 845 | Creates a tracked temporary file using ``vistir.compat.NamedTemporaryFile``, but creates 846 | a ``weakref.finalize`` call which will detach on garbage collection to close and delete 847 | the file. 848 | 849 | .. code:: python 850 | 851 | >>> temp_file = vistir.path.create_tracked_tempfile(prefix="requirements", suffix="txt") 852 | >>> temp_file.write("some\nstuff") 853 | >>> exit() 854 | 855 | 856 | .. _`path_to_url`: 857 | 858 | **path_to_url** 859 | //////////////// 860 | 861 | Convert the supplied local path to a file uri. 862 | 863 | .. code:: python 864 | 865 | >>> path_to_url("/home/user/code/myrepo/myfile.zip") 866 | 'file:///home/user/code/myrepo/myfile.zip' 867 | 868 | 869 | .. _`rmtree`: 870 | 871 | **rmtree** 872 | /////////// 873 | 874 | Stand-in for ``shutil.rmtree`` with additional error-handling. 875 | 876 | This version of `rmtree` handles read-only paths, especially in the case of index files 877 | written by certain source control systems. 878 | 879 | .. code:: python 880 | 881 | >>> vistir.path.rmtree('/tmp/test_dir') 882 | >>> [d for d in os.listdir('/tmp') if 'test_dir' in d] 883 | [] 884 | 885 | .. note:: 886 | 887 | Setting `ignore_errors=True` may cause this to silently fail to delete the path 888 | 889 | 890 | .. _`safe_expandvars`: 891 | 892 | **safe_expandvars** 893 | //////////////////// 894 | 895 | Call ``os.path.expandvars`` if value is a string, otherwise do nothing. 896 | 897 | .. code:: python 898 | 899 | >>> os.environ['TEST_VAR'] = "MY_TEST_VALUE" 900 | >>> vistir.path.safe_expandvars("https://myuser:${TEST_VAR}@myfakewebsite.com") 901 | 'https://myuser:MY_TEST_VALUE@myfakewebsite.com' 902 | 903 | 904 | .. _`set_write_bit`: 905 | 906 | **set_write_bit** 907 | ////////////////// 908 | 909 | Set read-write permissions for the current user on the target path. Fail silently 910 | if the path doesn't exist. 911 | 912 | .. code:: python 913 | 914 | >>> vistir.path.set_write_bit('/path/to/some/file') 915 | >>> with open('/path/to/some/file', 'w') as fh: 916 | fh.write("test text!") 917 | 918 | 919 | .. _`url_to_path`: 920 | 921 | **url_to_path** 922 | //////////////// 923 | 924 | Convert a valid file url to a local filesystem path. Follows logic taken from pip. 925 | 926 | .. code:: python 927 | 928 | >>> vistir.path.url_to_path("file:///home/user/somefile.zip") 929 | '/home/user/somefile.zip' 930 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = vistir 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import codecs 16 | import os 17 | import re 18 | import sys 19 | docs_dir = os.path.abspath(os.path.dirname(__file__)) 20 | src_dir = os.path.join(os.path.dirname(docs_dir), "src", "vistir") 21 | sys.path.insert(0, src_dir) 22 | version_file = os.path.join(src_dir, "__init__.py") 23 | 24 | 25 | def read_file(path): 26 | # intentionally *not* adding an encoding option to open, See: 27 | # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 28 | with codecs.open(path, 'r') as fp: 29 | return fp.read() 30 | 31 | 32 | def find_version(file_path): 33 | version_file = read_file(file_path) 34 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 35 | version_file, re.M) 36 | if version_match: 37 | return version_match.group(1) 38 | return '0.0.0' 39 | 40 | 41 | # -- Project information ----------------------------------------------------- 42 | 43 | project = 'vistir' 44 | copyright = '2018, Dan Ryan ' 45 | author = 'Dan Ryan ' 46 | 47 | release = find_version(version_file) 48 | version = '.'.join(release.split('.')[:2]) 49 | # The short X.Y version 50 | # version = '0.0' 51 | # The full version, including alpha/beta/rc tags 52 | # release = '0.0.0.dev0' 53 | 54 | 55 | # -- General configuration --------------------------------------------------- 56 | 57 | # If your documentation needs a minimal Sphinx version, state it here. 58 | # 59 | # needs_sphinx = '1.0' 60 | 61 | # Add any Sphinx extension module names here, as strings. They can be 62 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 63 | # ones. 64 | extensions = [ 65 | 'sphinx.ext.autodoc', 66 | 'sphinx.ext.viewcode', 67 | 'sphinx.ext.todo', 68 | 'sphinx.ext.intersphinx', 69 | 'sphinx.ext.autosummary' 70 | ] 71 | 72 | # Add any paths that contain templates here, relative to this directory. 73 | templates_path = ['_templates'] 74 | 75 | # The suffix(es) of source filenames. 76 | # You can specify multiple suffix as a list of string: 77 | # 78 | # source_suffix = ['.rst', '.md'] 79 | source_suffix = '.rst' 80 | 81 | # The master toctree document. 82 | master_doc = 'index' 83 | 84 | # The language for content autogenerated by Sphinx. Refer to documentation 85 | # for a list of supported languages. 86 | # 87 | # This is also used if you do content translation via gettext catalogs. 88 | # Usually you set "language" from the command line for these cases. 89 | language = 'en' 90 | 91 | # List of patterns, relative to source directory, that match files and 92 | # directories to ignore when looking for source files. 93 | # This pattern also affects html_static_path and html_extra_path . 94 | exclude_patterns = ['_build', '_man', 'Thumbs.db', '.DS_Store'] 95 | 96 | # The name of the Pygments (syntax highlighting) style to use. 97 | pygments_style = 'sphinx' 98 | autosummary_generate = True 99 | 100 | 101 | # -- Options for HTML output ------------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | # 106 | html_theme = 'sphinx_rtd_theme' 107 | 108 | # Theme options are theme-specific and customize the look and feel of a theme 109 | # further. For a list of options available for each theme, see the 110 | # documentation. 111 | # 112 | # html_theme_options = {} 113 | 114 | # Add any paths that contain custom static files (such as style sheets) here, 115 | # relative to this directory. They are copied after the builtin static files, 116 | # so a file named "default.css" will overwrite the builtin "default.css". 117 | html_static_path = ['_static'] 118 | 119 | # Custom sidebar templates, must be a dictionary that maps document names 120 | # to template names. 121 | # 122 | # The default sidebars (for documents that don't match any pattern) are 123 | # defined by theme itself. Builtin themes are using these templates by 124 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 125 | # 'searchbox.html']``. 126 | # 127 | # html_sidebars = {} 128 | 129 | 130 | # -- Options for HTMLHelp output --------------------------------------------- 131 | 132 | # Output file base name for HTML help builder. 133 | htmlhelp_basename = 'vistirdoc' 134 | extlinks = { 135 | 'issue': ('https://github.com/sarugaku/vistir/issues/%s', '#'), 136 | 'pull': ('https://github.com/sarugaku/vistir/pull/%s', 'PR #'), 137 | } 138 | html_theme_options = { 139 | 'display_version': True, 140 | 'prev_next_buttons_location': 'bottom', 141 | 'style_external_links': True, 142 | 'vcs_pageview_mode': '', 143 | # Toc options 144 | 'collapse_navigation': True, 145 | 'sticky_navigation': True, 146 | 'navigation_depth': 4, 147 | 'includehidden': True, 148 | 'titles_only': False 149 | } 150 | 151 | # -- Options for LaTeX output ------------------------------------------------ 152 | 153 | latex_elements = { 154 | # The paper size ('letterpaper' or 'a4paper'). 155 | # 156 | 'papersize': 'letterpaper', 157 | 158 | # The font size ('10pt', '11pt' or '12pt'). 159 | # 160 | 'pointsize': '10pt', 161 | 162 | # Additional stuff for the LaTeX preamble. 163 | # 164 | # 'preamble': '', 165 | 166 | # Latex figure (float) alignment 167 | # 168 | # 'figure_align': 'htbp', 169 | } 170 | 171 | # Grouping the document tree into LaTeX files. List of tuples 172 | # (source start file, target name, title, 173 | # author, documentclass [howto, manual, or own class]). 174 | latex_documents = [ 175 | (master_doc, 'vistir.tex', 'vistir Documentation', 176 | 'Dan Ryan \\textless{}dan@danryan.co\\textgreater{}', 'manual'), 177 | ] 178 | 179 | 180 | # -- Options for manual page output ------------------------------------------ 181 | 182 | # One entry per manual page. List of tuples 183 | # (source start file, name, description, authors, manual section). 184 | man_pages = [ 185 | (master_doc, 'vistir', 'vistir Documentation', 186 | [author], 1) 187 | ] 188 | 189 | 190 | # -- Options for Texinfo output ---------------------------------------------- 191 | 192 | # Grouping the document tree into Texinfo files. List of tuples 193 | # (source start file, target name, title, author, 194 | # dir menu entry, description, category) 195 | texinfo_documents = [ 196 | (master_doc, 'vistir', 'vistir Documentation', 197 | author, 'vistir', 'Miscellaneous utilities for dealing with filesystems, paths, projects, subprocesses, and more.', 198 | 'Miscellaneous'), 199 | ] 200 | 201 | 202 | # -- Options for Epub output ------------------------------------------------- 203 | 204 | # Bibliographic Dublin Core info. 205 | epub_title = project 206 | epub_author = author 207 | epub_publisher = author 208 | epub_copyright = copyright 209 | 210 | # The unique identifier of the text. This can be a ISBN number 211 | # or the project homepage. 212 | # 213 | # epub_identifier = '' 214 | 215 | # A unique identification for the text. 216 | # 217 | # epub_uid = '' 218 | 219 | # A list of files that should not be packed into the epub file. 220 | epub_exclude_files = ['search.html'] 221 | 222 | 223 | # -- Extension configuration ------------------------------------------------- 224 | 225 | # -- Options for todo extension ---------------------------------------------- 226 | 227 | # If true, `todo` and `todoList` produce output, else they produce nothing. 228 | todo_include_todos = True 229 | intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} 230 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. vistir documentation master file, created by 2 | sphinx-quickstart on Sat Aug 11 02:27:14 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to vistir's documentation! 7 | ================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 4 11 | :caption: Contents: 12 | 13 | quickstart 14 | vistir 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=vistir 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | =============================================================================== 2 | vistir: Setup / utilities which most projects eventually need 3 | =============================================================================== 4 | 5 | .. image:: https://img.shields.io/pypi/v/vistir.svg 6 | :target: https://pypi.python.org/pypi/vistir 7 | 8 | .. image:: https://img.shields.io/pypi/l/vistir.svg 9 | :target: https://pypi.python.org/pypi/vistir 10 | 11 | .. image:: https://travis-ci.com/sarugaku/vistir.svg?branch=master 12 | :target: https://travis-ci.com/sarugaku/vistir 13 | 14 | .. image:: https://dev.azure.com/sarugaku/vistir/_apis/build/status/Vistir%20Build%20Pipeline?branchName=master 15 | :target: https://dev.azure.com/sarugaku/vistir/_build/latest?definitionId=2&branchName=master 16 | 17 | .. image:: https://img.shields.io/pypi/pyversions/vistir.svg 18 | :target: https://pypi.python.org/pypi/vistir 19 | 20 | .. image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg 21 | :target: https://saythanks.io/to/techalchemy 22 | 23 | .. image:: https://readthedocs.org/projects/vistir/badge/?version=latest 24 | :target: https://vistir.readthedocs.io/en/latest/?badge=latest 25 | :alt: Documentation Status 26 | 27 | 28 | 🐉 Installation 29 | ================= 30 | 31 | Install from `PyPI`_: 32 | 33 | :: 34 | 35 | $ pipenv install vistir 36 | 37 | Install from `Github`_: 38 | 39 | :: 40 | 41 | $ pipenv install -e git+https://github.com/sarugaku/vistir.git#egg=vistir 42 | 43 | 44 | .. _PyPI: https://www.pypi.org/project/vistir 45 | .. _Github: https://github.com/sarugaku/vistir 46 | 47 | 48 | .. _`Summary`: 49 | 50 | 🐉 Summary 51 | =========== 52 | 53 | **vistir** is a library full of utility functions designed to make life easier. Here are 54 | some of the places where these functions are used: 55 | 56 | * `pipenv`_ 57 | * `requirementslib`_ 58 | * `pip-tools`_ 59 | * `passa`_ 60 | * `pythonfinder`_ 61 | 62 | .. _passa: https://github.com/sarugaku/passa 63 | .. _pipenv: https://github.com/pypa/pipenv 64 | .. _pip-tools: https://github.com/jazzband/pip-tools 65 | .. _requirementslib: https://github.com/sarugaku/requirementslib 66 | .. _pythonfinder: https://github.com/sarugaku/pythonfinder 67 | 68 | 69 | .. _`Usage`: 70 | 71 | 🐉 Usage 72 | ========== 73 | 74 | Importing a utility 75 | -------------------- 76 | 77 | You can import utilities directly from **vistir**: 78 | 79 | .. code:: python 80 | 81 | from vistir import cd 82 | cd('/path/to/somedir'): 83 | do_stuff_in('somedir') 84 | 85 | 86 | .. _`Functionality`: 87 | 88 | 🐉 Functionality 89 | ================== 90 | 91 | **vistir** provides several categories of functionality, including: 92 | 93 | * Backports 94 | * Compatibility Shims 95 | * Context Managers 96 | * Miscellaneous Utilities 97 | * Path Utilities 98 | 99 | .. note:: 100 | 101 | The backports should be imported via :mod:`~vistir.compat` which will provide the 102 | native versions of the backported items if possible. 103 | 104 | 🐉 Context Managers 105 | -------------------- 106 | 107 | **vistir** provides the following context managers as utility contexts: 108 | 109 | * :func:`~vistir.contextmanagers.atomic_open_for_write` 110 | * :func:`~vistir.contextmanagers.cd` 111 | * :func:`~vistir.contextmanagers.open_file` 112 | * :func:`~vistir.contextmanagers.replaced_stream` 113 | * :func:`~vistir.contextmanagers.replaced_streams` 114 | * :func:`~vistir.contextmanagers.spinner` 115 | * :func:`~vistir.contextmanagers.temp_environ` 116 | * :func:`~vistir.contextmanagers.temp_path` 117 | 118 | 119 | .. _`atomic_open_for_write`: 120 | 121 | **atomic_open_for_write** 122 | /////////////////////////// 123 | 124 | This context manager ensures that a file only gets overwritten if the contents can be 125 | successfully written in its place. If you open a file for writing and then fail in the 126 | middle under normal circumstances, your original file is already gone. 127 | 128 | .. code:: python 129 | 130 | >>> fn = "test_file.txt" 131 | >>> with open(fn, "w") as fh: 132 | fh.write("this is some test text") 133 | >>> read_test_file() 134 | this is some test text 135 | >>> def raise_exception_while_writing(filename): 136 | with vistir.contextmanagers.atomic_open_for_write(filename) as fh: 137 | fh.write("Overwriting all the text from before with even newer text") 138 | raise RuntimeError("But did it get overwritten now?") 139 | >>> raise_exception_while_writing(fn) 140 | Traceback (most recent call last): 141 | ... 142 | RuntimeError: But did it get overwritten now? 143 | >>> read_test_file() 144 | writing some new text 145 | 146 | 147 | .. _`cd`: 148 | 149 | **cd** 150 | /////// 151 | 152 | A context manager for temporarily changing the working directory. 153 | 154 | 155 | .. code:: python 156 | 157 | >>> os.path.abspath(os.curdir) 158 | '/tmp/test' 159 | >>> with vistir.contextmanagers.cd('/tmp/vistir_test'): 160 | print(os.path.abspath(os.curdir)) 161 | /tmp/vistir_test 162 | 163 | 164 | .. _`open_file`: 165 | 166 | **open_file** 167 | /////////////// 168 | 169 | A context manager for streaming file contents, either local or remote. It is recommended 170 | to pair this with an iterator which employs a sensible chunk size. 171 | 172 | 173 | .. code:: python 174 | 175 | >>> filecontents = b"" 176 | with vistir.contextmanagers.open_file("https://norvig.com/big.txt") as fp: 177 | for chunk in iter(lambda: fp.read(16384), b""): 178 | filecontents.append(chunk) 179 | >>> import io 180 | >>> import shutil 181 | >>> filecontents = io.BytesIO(b"") 182 | >>> with vistir.contextmanagers.open_file("https://norvig.com/big.txt") as fp: 183 | shutil.copyfileobj(fp, filecontents) 184 | 185 | 186 | .. _`replaced_stream`: 187 | 188 | **replaced_stream** 189 | //////////////////// 190 | 191 | A context manager to temporarily swap out *stream_name* with a stream wrapper. This will 192 | capture the stream output and prevent it from being written as normal. 193 | 194 | 195 | .. code-block:: python 196 | 197 | >>> orig_stdout = sys.stdout 198 | >>> with replaced_stream("stdout") as stdout: 199 | ... sys.stdout.write("hello") 200 | ... assert stdout.getvalue() == "hello" 201 | 202 | >>> sys.stdout.write("hello") 203 | 'hello' 204 | 205 | 206 | .. _`replaced_streams`: 207 | 208 | **replaced_streams** 209 | ///////////////////// 210 | 211 | 212 | Temporarily replaces both *sys.stdout* and *sys.stderr* and captures anything written 213 | to these respective targets. 214 | 215 | 216 | .. code-block:: python 217 | 218 | >>> import sys 219 | >>> with vistir.contextmanagers.replaced_streams() as streams: 220 | >>> stdout, stderr = streams 221 | >>> sys.stderr.write("test") 222 | >>> sys.stdout.write("hello") 223 | >>> assert stdout.getvalue() == "hello" 224 | >>> assert stderr.getvalue() == "test" 225 | 226 | >>> stdout.getvalue() 227 | 'hello' 228 | 229 | >>> stderr.getvalue() 230 | 'test' 231 | 232 | 233 | .. _`spinner`: 234 | 235 | **spinner** 236 | //////////// 237 | 238 | A context manager for wrapping some actions with a threaded, interrupt-safe spinner. The 239 | spinner is fully compatible with all terminals (you can use ``bouncingBar`` on non-utf8 240 | terminals) and will allow you to update the text of the spinner itself by simply setting 241 | ``spinner.text`` or write lines to the screen above the spinner by using 242 | ``spinner.write(line)``. Success text can be indicated using ``spinner.ok("Text")`` and 243 | failure text can be indicated with ``spinner.fail("Fail text")``. 244 | 245 | .. code:: python 246 | 247 | >>> lines = ["a", "b"] 248 | >>> with vistir.contextmanagers.spinner(spinner_name="dots", text="Running...", handler_map={}, nospin=False) as sp: 249 | for line in lines: 250 | sp.write(line + "\n") 251 | while some_variable = some_queue.pop(): 252 | sp.text = "Consuming item: %s" % some_variable 253 | if success_condition: 254 | sp.ok("Succeeded!") 255 | else: 256 | sp.fail("Failed!") 257 | 258 | 259 | .. _`temp_environ`: 260 | 261 | **temp_environ** 262 | ///////////////// 263 | 264 | Sets a temporary environment context to freely manipulate :data:`os.environ` which will 265 | be reset upon exiting the context. 266 | 267 | 268 | .. code:: python 269 | 270 | >>> os.environ['MY_KEY'] = "test" 271 | >>> os.environ['MY_KEY'] 272 | 'test' 273 | >>> with vistir.contextmanagers.temp_environ(): 274 | os.environ['MY_KEY'] = "another thing" 275 | print("New key: %s" % os.environ['MY_KEY']) 276 | New key: another thing 277 | >>> os.environ['MY_KEY'] 278 | 'test' 279 | 280 | 281 | .. _`temp_path`: 282 | 283 | **temp_path** 284 | ////////////// 285 | 286 | Sets a temporary environment context to freely manipulate :data:`sys.path` which will 287 | be reset upon exiting the context. 288 | 289 | 290 | .. code:: python 291 | 292 | >>> path_from_virtualenv = load_path("/path/to/venv/bin/python") 293 | >>> print(sys.path) 294 | ['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'] 295 | >>> with temp_path(): 296 | sys.path = path_from_virtualenv 297 | # Running in the context of the path above 298 | run(["pip", "install", "stuff"]) 299 | >>> print(sys.path) 300 | ['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'] 301 | 302 | 303 | 🐉 Miscellaneous Utilities 304 | -------------------------- 305 | 306 | The following Miscellaneous utilities are available as helper methods: 307 | 308 | * :func:`~vistir.misc.shell_escape` 309 | * :func:`~vistir.misc.unnest` 310 | * :func:`~vistir.misc.dedup` 311 | * :func:`~vistir.misc.run` 312 | * :func:`~vistir.misc.load_path` 313 | * :func:`~vistir.misc.partialclass` 314 | * :func:`~vistir.misc.to_text` 315 | * :func:`~vistir.misc.to_bytes` 316 | * :func:`~vistir.misc.divide` 317 | * :func:`~vistir.misc.take` 318 | * :func:`~vistir.misc.chunked` 319 | * :func:`~vistir.misc.decode_for_output` 320 | * :func:`~vistir.misc.get_canonical_encoding_name` 321 | * :func:`~vistir.misc.get_wrapped_stream` 322 | * :class:`~vistir.misc.StreamWrapper` 323 | * :func:`~vistir.misc.get_text_stream` 324 | * :func:`~vistir.misc.replace_with_text_stream` 325 | * :func:`~vistir.misc.get_text_stdin` 326 | * :func:`~vistir.misc.get_text_stdout` 327 | * :func:`~vistir.misc.get_text_stderr` 328 | * :func:`~vistir.misc.echo` 329 | 330 | 331 | .. _`shell_escape`: 332 | 333 | **shell_escape** 334 | ///////////////// 335 | 336 | Escapes a string for use as shell input when passing *shell=True* to :func:`os.Popen`. 337 | 338 | .. code:: python 339 | 340 | >>> vistir.misc.shell_escape("/tmp/test/test script.py hello") 341 | '/tmp/test/test script.py hello' 342 | 343 | 344 | .. _`unnest`: 345 | 346 | **unnest** 347 | /////////// 348 | 349 | Unnests nested iterables into a flattened one. 350 | 351 | .. code:: python 352 | 353 | >>> nested_iterable = (1234, (3456, 4398345, (234234)), (2396, (23895750, 9283798, 29384, (289375983275, 293759, 2347, (2098, 7987, 27599))))) 354 | >>> list(vistir.misc.unnest(nested_iterable)) 355 | [1234, 3456, 4398345, 234234, 2396, 23895750, 9283798, 29384, 289375983275, 293759, 2347, 2098, 7987, 27599] 356 | 357 | 358 | .. _`dedup`: 359 | 360 | **dedup** 361 | ////////// 362 | 363 | Deduplicates an iterable (like a :class:`set`, but preserving order). 364 | 365 | .. code:: python 366 | 367 | >>> iterable = ["repeatedval", "uniqueval", "repeatedval", "anotherval", "somethingelse"] 368 | >>> list(vistir.misc.dedup(iterable)) 369 | ['repeatedval', 'uniqueval', 'anotherval', 'somethingelse'] 370 | 371 | .. _`run`: 372 | 373 | **run** 374 | //////// 375 | 376 | Runs the given command using :func:`subprocess.Popen` and passing sane defaults. 377 | 378 | .. code:: python 379 | 380 | >>> out, err = vistir.run(["cat", "/proc/version"]) 381 | >>> out 382 | 'Linux version 4.15.0-27-generic (buildd@lgw01-amd64-044) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #29-Ubuntu SMP Wed Jul 11 08:21:57 UTC 2018' 383 | 384 | 385 | .. _`load_path`: 386 | 387 | **load_path** 388 | ////////////// 389 | 390 | Load the :data:`sys.path` from the given python executable's environment as json. 391 | 392 | .. code:: python 393 | 394 | >>> load_path("/home/user/.virtualenvs/requirementslib-5MhGuG3C/bin/python") 395 | ['', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python37.zip', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/site-packages', '/home/user/git/requirementslib/src'] 396 | 397 | 398 | .. _`partialclass`: 399 | 400 | **partialclass** 401 | ///////////////// 402 | 403 | Create a partially instantiated class. 404 | 405 | .. code:: python 406 | 407 | >>> source = partialclass(Source, url="https://pypi.org/simple") 408 | >>> new_source = source(name="pypi") 409 | >>> new_source 410 | <__main__.Source object at 0x7f23af189b38> 411 | >>> new_source.__dict__ 412 | {'url': 'https://pypi.org/simple', 'verify_ssl': True, 'name': 'pypi'} 413 | 414 | 415 | .. _`to_text`: 416 | 417 | **to_text** 418 | //////////// 419 | 420 | Convert arbitrary text-formattable input to text while handling errors. 421 | 422 | .. code:: python 423 | 424 | >>> vistir.misc.to_text(b"these are bytes") 425 | 'these are bytes' 426 | 427 | 428 | .. _`to_bytes`: 429 | 430 | **to_bytes** 431 | ///////////// 432 | 433 | Converts arbitrary byte-convertable input to bytes while handling errors. 434 | 435 | .. code:: python 436 | 437 | >>> vistir.misc.to_bytes("this is some text") 438 | b'this is some text' 439 | >>> vistir.misc.to_bytes(u"this is some text") 440 | b'this is some text' 441 | 442 | 443 | .. _`chunked`: 444 | 445 | **chunked** 446 | //////////// 447 | 448 | Splits an iterable up into groups *of the specified length*, per `more itertools`_. Returns an iterable. 449 | 450 | This example will create groups of chunk size **5**, which means there will be *6 groups*. 451 | 452 | .. code-block:: python 453 | 454 | >>> chunked_iterable = vistir.misc.chunked(5, range(30)) 455 | >>> for chunk in chunked_iterable: 456 | ... add_to_some_queue(chunk) 457 | 458 | .. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping 459 | 460 | 461 | .. _`take`: 462 | 463 | **take** 464 | ///////// 465 | 466 | Take elements from the supplied iterable without consuming it. 467 | 468 | .. code-block:: python 469 | 470 | >>> iterable = range(30) 471 | >>> first_10 = take(10, iterable) 472 | >>> [i for i in first_10] 473 | [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] 474 | 475 | >>> [i for i in iterable] 476 | [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] 477 | 478 | 479 | .. _`divide`: 480 | 481 | **divide** 482 | //////////// 483 | 484 | Splits an iterable up into the *specified number of groups*, per `more itertools`_. Returns an iterable. 485 | 486 | .. code-block:: python 487 | 488 | >>> iterable = range(30) 489 | >>> groups = [] 490 | >>> for grp in vistir.misc.divide(3, iterable): 491 | ... groups.append(grp) 492 | >>> groups 493 | [, , ] 494 | 495 | 496 | .. _more itertools: https://more-itertools.readthedocs.io/en/latest/api.html#grouping 497 | 498 | 499 | .. _`decode_for_output`: 500 | 501 | **decode_for_output** 502 | ////////////////////// 503 | 504 | Converts an arbitrary text input to output which is encoded for printing to terminal 505 | outputs using the system preferred locale using ``locale.getpreferredencoding(False)`` 506 | with some additional hackery on linux systems. 507 | 508 | .. code:: python 509 | 510 | >>> vistir.misc.decode_for_output(u"Some text") 511 | "some default locale encoded text" 512 | 513 | 514 | .. _`get_canonical_encoding_name`: 515 | 516 | **get_canonical_encoding_name** 517 | //////////////////////////////// 518 | 519 | Given an encoding name, get the canonical name from a codec lookup. 520 | 521 | .. code-block:: python 522 | 523 | >>> vistir.misc.get_canonical_encoding_name("utf8") 524 | "utf-8" 525 | 526 | 527 | .. _`get_wrapped_stream`: 528 | 529 | **get_wrapped_stream** 530 | ////////////////////// 531 | 532 | Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. 533 | 534 | .. code-block:: python 535 | 536 | >>> stream = sys.stdout 537 | >>> wrapped_stream = vistir.misc.get_wrapped_stream(sys.stdout) 538 | 539 | 540 | .. _`StreamWrapper`: 541 | 542 | **StreamWrapper** 543 | ////////////////// 544 | 545 | A stream wrapper and compatibility class for handling wrapping file-like stream objects 546 | which may be used in place of ``sys.stdout`` and other streams. 547 | 548 | .. code-block:: python 549 | 550 | >>> wrapped_stream = vistir.misc.StreamWrapper(sys.stdout, encoding="utf-8", errors="replace", line_buffering=True) 551 | >>> wrapped_stream = vistir.misc.StreamWrapper(io.StringIO(), encoding="utf-8", errors="replace", line_buffering=True) 552 | 553 | 554 | .. _`get_text_stream`: 555 | 556 | **get_text_stream** 557 | //////////////////// 558 | 559 | An implementation of the **StreamWrapper** for the purpose of wrapping **sys.stdin** or **sys.stdout**. 560 | 561 | On Windows, this returns the appropriate handle to the requested output stream. 562 | 563 | .. code-block:: python 564 | 565 | >>> text_stream = vistir.misc.get_text_stream("stdout") 566 | >>> sys.stdout = text_stream 567 | >>> sys.stdin = vistir.misc.get_text_stream("stdin") 568 | >>> vistir.misc.echo(u"\0499", fg="green") 569 | ҙ 570 | 571 | 572 | .. _`replace_with_text_stream`: 573 | 574 | **replace_with_text_stream** 575 | ///////////////////////////// 576 | 577 | Given a text stream name, replaces the text stream with a **StreamWrapper** instance. 578 | 579 | 580 | .. code-block:: python 581 | 582 | >>> vistir.misc.replace_with_text_stream("stdout") 583 | 584 | Once invoked, the standard stream in question is replaced with the required wrapper, 585 | turning it into a ``TextIOWrapper`` compatible stream (which ensures that unicode 586 | characters can be written to it). 587 | 588 | 589 | .. _`get_text_stdin`: 590 | 591 | **get_text_stdin** 592 | /////////////////// 593 | 594 | A helper function for calling **get_text_stream("stdin")**. 595 | 596 | 597 | .. _`get_text_stdout`: 598 | 599 | **get_text_stdout** 600 | //////////////////// 601 | 602 | A helper function for calling **get_text_stream("stdout")**. 603 | 604 | 605 | .. _`get_text_stderr`: 606 | 607 | **get_text_stderr** 608 | //////////////////// 609 | 610 | A helper function for calling **get_text_stream("stderr")**. 611 | 612 | 613 | .. _`echo`: 614 | 615 | **echo** 616 | ///////// 617 | 618 | Writes colored, stream-compatible output to the desired handle (``sys.stdout`` by default). 619 | 620 | .. code-block:: python 621 | 622 | >>> vistir.misc.echo("some text", fg="green", bg="black", style="bold", err=True) # write to stderr 623 | some text 624 | >>> vistir.misc.echo("some other text", fg="cyan", bg="white", style="underline") # write to stdout 625 | some other text 626 | 627 | 628 | 🐉 Path Utilities 629 | ------------------ 630 | 631 | **vistir** provides utilities for interacting with filesystem paths: 632 | 633 | * :func:`vistir.path.normalize_path` 634 | * :func:`vistir.path.is_in_path` 635 | * :func:`vistir.path.get_converted_relative_path` 636 | * :func:`vistir.path.handle_remove_readonly` 637 | * :func:`vistir.path.is_file_url` 638 | * :func:`vistir.path.is_readonly_path` 639 | * :func:`vistir.path.is_valid_url` 640 | * :func:`vistir.path.mkdir_p` 641 | * :func:`vistir.path.ensure_mkdir_p` 642 | * :func:`vistir.path.create_tracked_tempdir` 643 | * :func:`vistir.path.create_tracked_tempfile` 644 | * :func:`vistir.path.path_to_url` 645 | * :func:`vistir.path.rmtree` 646 | * :func:`vistir.path.safe_expandvars` 647 | * :func:`vistir.path.set_write_bit` 648 | * :func:`vistir.path.url_to_path` 649 | * :func:`vistir.path.walk_up` 650 | 651 | 652 | .. _`normalize_path`: 653 | 654 | **normalize_path** 655 | ////////////////// 656 | 657 | Return a case-normalized absolute variable-expanded path. 658 | 659 | 660 | .. code:: python 661 | 662 | >>> vistir.path.normalize_path("~/${USER}") 663 | /home/user/user 664 | 665 | 666 | .. _`is_in_path`: 667 | 668 | **is_in_path** 669 | ////////////// 670 | 671 | Determine if the provided full path is in the given parent root. 672 | 673 | 674 | .. code:: python 675 | 676 | >>> vistir.path.is_in_path("~/.pyenv/versions/3.7.1/bin/python", "${PYENV_ROOT}/versions") 677 | True 678 | 679 | 680 | .. _`get_converted_relative_path`: 681 | 682 | **get_converted_relative_path** 683 | //////////////////////////////// 684 | 685 | Convert the supplied path to a relative path (relative to :data:`os.curdir`) 686 | 687 | 688 | .. code:: python 689 | 690 | >>> os.chdir('/home/user/code/myrepo/myfolder') 691 | >>> vistir.path.get_converted_relative_path('/home/user/code/file.zip') 692 | './../../file.zip' 693 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder/mysubfolder') 694 | './mysubfolder' 695 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') 696 | '.' 697 | 698 | 699 | .. _`handle_remove_readonly`: 700 | 701 | **handle_remove_readonly** 702 | /////////////////////////// 703 | 704 | Error handler for shutil.rmtree. 705 | 706 | Windows source repo folders are read-only by default, so this error handler attempts to 707 | set them as writeable and then proceed with deletion. 708 | 709 | This function will call check :func:`vistir.path.is_readonly_path` before attempting to 710 | call :func:`vistir.path.set_write_bit` on the target path and try again. 711 | 712 | 713 | .. _`is_file_url`: 714 | 715 | **is_file_url** 716 | //////////////// 717 | 718 | Checks whether the given url is a properly formatted ``file://`` uri. 719 | 720 | .. code:: python 721 | 722 | >>> vistir.path.is_file_url('file:///home/user/somefile.zip') 723 | True 724 | >>> vistir.path.is_file_url('/home/user/somefile.zip') 725 | False 726 | 727 | 728 | .. _`is_readonly_path`: 729 | 730 | **is_readonly_path** 731 | ///////////////////// 732 | 733 | Check if a provided path exists and is readonly by checking for ``bool(path.stat & stat.S_IREAD) and not os.access(path, os.W_OK)`` 734 | 735 | .. code:: python 736 | 737 | >>> vistir.path.is_readonly_path('/etc/passwd') 738 | True 739 | >>> vistir.path.is_readonly_path('/home/user/.bashrc') 740 | False 741 | 742 | 743 | .. _`is_valid_url`: 744 | 745 | **is_valid_url** 746 | ///////////////// 747 | 748 | Checks whether a URL is valid and parseable by checking for the presence of a scheme and 749 | a netloc. 750 | 751 | .. code:: python 752 | 753 | >>> vistir.path.is_valid_url("https://google.com") 754 | True 755 | >>> vistir.path.is_valid_url("/home/user/somefile") 756 | False 757 | 758 | 759 | .. _`mkdir_p`: 760 | 761 | **mkdir_p** 762 | ///////////// 763 | 764 | Recursively creates the target directory and all of its parents if they do not 765 | already exist. Fails silently if they do. 766 | 767 | .. code:: python 768 | 769 | >>> os.mkdir('/tmp/test_dir') 770 | >>> os.listdir('/tmp/test_dir') 771 | [] 772 | >>> vistir.path.mkdir_p('/tmp/test_dir/child/subchild/subsubchild') 773 | >>> os.listdir('/tmp/test_dir/child/subchild') 774 | ['subsubchild'] 775 | 776 | 777 | .. _`ensure_mkdir_p`: 778 | 779 | **ensure_mkdir_p** 780 | /////////////////// 781 | 782 | A decorator which ensures that the caller function's return value is created as a 783 | directory on the filesystem. 784 | 785 | .. code:: python 786 | 787 | >>> @ensure_mkdir_p 788 | def return_fake_value(path): 789 | return path 790 | >>> return_fake_value('/tmp/test_dir') 791 | >>> os.listdir('/tmp/test_dir') 792 | [] 793 | >>> return_fake_value('/tmp/test_dir/child/subchild/subsubchild') 794 | >>> os.listdir('/tmp/test_dir/child/subchild') 795 | ['subsubchild'] 796 | 797 | 798 | .. _`create_tracked_tempdir`: 799 | 800 | **create_tracked_tempdir** 801 | //////////////////////////// 802 | 803 | Creates a tracked temporary directory using :class:`~vistir.path.TemporaryDirectory`, but does 804 | not remove the directory when the return value goes out of scope, instead registers a 805 | handler to cleanup on program exit. 806 | 807 | .. code:: python 808 | 809 | >>> temp_dir = vistir.path.create_tracked_tempdir(prefix="test_dir") 810 | >>> assert temp_dir.startswith("test_dir") 811 | True 812 | >>> with vistir.path.create_tracked_tempdir(prefix="test_dir") as temp_dir: 813 | with io.open(os.path.join(temp_dir, "test_file.txt"), "w") as fh: 814 | fh.write("this is a test") 815 | >>> os.listdir(temp_dir) 816 | 817 | 818 | .. _`create_tracked_tempfile`: 819 | 820 | **create_tracked_tempfile** 821 | //////////////////////////// 822 | 823 | Creates a tracked temporary file using ``vistir.compat.NamedTemporaryFile``, but creates 824 | a ``weakref.finalize`` call which will detach on garbage collection to close and delete 825 | the file. 826 | 827 | .. code:: python 828 | 829 | >>> temp_file = vistir.path.create_tracked_tempfile(prefix="requirements", suffix="txt") 830 | >>> temp_file.write("some\nstuff") 831 | >>> exit() 832 | 833 | 834 | .. _`path_to_url`: 835 | 836 | **path_to_url** 837 | //////////////// 838 | 839 | Convert the supplied local path to a file uri. 840 | 841 | .. code:: python 842 | 843 | >>> path_to_url("/home/user/code/myrepo/myfile.zip") 844 | 'file:///home/user/code/myrepo/myfile.zip' 845 | 846 | 847 | .. _`rmtree`: 848 | 849 | **rmtree** 850 | /////////// 851 | 852 | Stand-in for :func:`~shutil.rmtree` with additional error-handling. 853 | 854 | This version of `rmtree` handles read-only paths, especially in the case of index files 855 | written by certain source control systems. 856 | 857 | .. code:: python 858 | 859 | >>> vistir.path.rmtree('/tmp/test_dir') 860 | >>> [d for d in os.listdir('/tmp') if 'test_dir' in d] 861 | [] 862 | 863 | .. note:: 864 | 865 | Setting `ignore_errors=True` may cause this to silently fail to delete the path 866 | 867 | 868 | .. _`safe_expandvars`: 869 | 870 | **safe_expandvars** 871 | //////////////////// 872 | 873 | Call :func:`os.path.expandvars` if value is a string, otherwise do nothing. 874 | 875 | .. code:: python 876 | 877 | >>> os.environ['TEST_VAR'] = "MY_TEST_VALUE" 878 | >>> vistir.path.safe_expandvars("https://myuser:${TEST_VAR}@myfakewebsite.com") 879 | 'https://myuser:MY_TEST_VALUE@myfakewebsite.com' 880 | 881 | 882 | .. _`set_write_bit`: 883 | 884 | **set_write_bit** 885 | ////////////////// 886 | 887 | Set read-write permissions for the current user on the target path. Fail silently 888 | if the path doesn't exist. 889 | 890 | .. code:: python 891 | 892 | >>> vistir.path.set_write_bit('/path/to/some/file') 893 | >>> with open('/path/to/some/file', 'w') as fh: 894 | fh.write("test text!") 895 | 896 | 897 | .. _`url_to_path`: 898 | 899 | **url_to_path** 900 | //////////////// 901 | 902 | Convert a valid file url to a local filesystem path. Follows logic taken from pip. 903 | 904 | .. code:: python 905 | 906 | >>> vistir.path.url_to_path("file:///home/user/somefile.zip") 907 | '/home/user/somefile.zip' 908 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx_rtd_theme 3 | -------------------------------------------------------------------------------- /docs/vistir.cmdparse.rst: -------------------------------------------------------------------------------- 1 | vistir.cmdparse module 2 | ====================== 3 | 4 | .. automodule:: vistir.cmdparse 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.compat.rst: -------------------------------------------------------------------------------- 1 | vistir.compat module 2 | ==================== 3 | 4 | .. automodule:: vistir.compat 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.contextmanagers.rst: -------------------------------------------------------------------------------- 1 | vistir.contextmanagers module 2 | ============================= 3 | 4 | .. automodule:: vistir.contextmanagers 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.cursor.rst: -------------------------------------------------------------------------------- 1 | vistir.cursor module 2 | ==================== 3 | 4 | .. automodule:: vistir.cursor 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.misc.rst: -------------------------------------------------------------------------------- 1 | vistir.misc module 2 | ================== 3 | 4 | .. automodule:: vistir.misc 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.path.rst: -------------------------------------------------------------------------------- 1 | vistir.path module 2 | ================== 3 | 4 | .. automodule:: vistir.path 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.rst: -------------------------------------------------------------------------------- 1 | vistir package 2 | ============== 3 | 4 | .. automodule:: vistir 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | Submodules 10 | ---------- 11 | 12 | .. toctree:: 13 | 14 | vistir.cmdparse 15 | vistir.compat 16 | vistir.contextmanagers 17 | vistir.misc 18 | vistir.path 19 | 20 | -------------------------------------------------------------------------------- /docs/vistir.spin.rst: -------------------------------------------------------------------------------- 1 | vistir.spin module 2 | ================== 3 | 4 | .. automodule:: vistir.spin 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/vistir.termcolors.rst: -------------------------------------------------------------------------------- 1 | vistir.termcolors module 2 | ======================== 3 | 4 | .. automodule:: vistir.termcolors 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /news/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools>=40.8.0', 'wheel>=0.33.0'] 3 | 4 | [tool.black] 5 | line-length = 90 6 | include = '\.pyi?$' 7 | exclude = ''' 8 | /( 9 | \.eggs 10 | | \.git 11 | | \.hg 12 | | \.mypy_cache 13 | | \.tox 14 | | \.pyre_configuration 15 | | \.venv 16 | | _build 17 | | buck-out 18 | | build 19 | | dist 20 | | src/vistir/compat.py 21 | ) 22 | ''' 23 | 24 | [tool.towncrier] 25 | package = 'vistir' 26 | package_dir = 'src' 27 | filename = 'CHANGELOG.rst' 28 | directory = 'news/' 29 | title_format = '{version} ({project_date})' 30 | issue_format = '`#{issue} `_' 31 | template = 'tasks/CHANGELOG.rst.jinja2' 32 | 33 | [[tool.towncrier.type]] 34 | directory = 'feature' 35 | name = 'Features' 36 | showcontent = true 37 | 38 | [[tool.towncrier.type]] 39 | directory = 'bugfix' 40 | name = 'Bug Fixes' 41 | showcontent = true 42 | 43 | [[tool.towncrier.type]] 44 | directory = 'trivial' 45 | name = 'Trivial Changes' 46 | showcontent = false 47 | 48 | [[tool.towncrier.type]] 49 | directory = 'removal' 50 | name = 'Removals and Deprecations' 51 | showcontent = true 52 | 53 | [[tool.towncrier.type]] 54 | directory = 'docs' 55 | name = 'Documentation Updates & Additions' 56 | showcontent = true 57 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = vistir 3 | description = Miscellaneous utilities for dealing with filesystems, paths, projects, subprocesses, and more. 4 | url = https://github.com/sarugaku/vistir 5 | author = Dan Ryan 6 | author_email = dan@danryan.co 7 | long_description = file: README.rst 8 | long_description_content_type = text/x-rst 9 | license = ISC License 10 | keywords = 11 | tools 12 | utilities 13 | backports 14 | paths 15 | subprocess 16 | filesystem 17 | 18 | classifier = 19 | Development Status :: 4 - Beta 20 | License :: OSI Approved :: ISC License (ISCL) 21 | Programming Language :: Python :: 3 22 | Programming Language :: Python :: 3.7 23 | Programming Language :: Python :: 3.8 24 | Programming Language :: Python :: 3.9 25 | Programming Language :: Python :: 3.10 26 | Programming Language :: Python :: 3.11 27 | Topic :: Software Development :: Libraries :: Python Modules 28 | 29 | [options] 30 | zip_safe = true 31 | python_requires = >=3.7 32 | setup_requires = 33 | setuptools>=40.8.0 34 | wheel 35 | 36 | install_requires = 37 | colorama>=0.3.4,!=0.4.2 38 | 39 | [options.extras_require] 40 | requests = 41 | requests 42 | dev = 43 | pre-commit 44 | coverage 45 | isort 46 | flake8 47 | flake8-bugbear;python_version>="3.5" 48 | rope 49 | black;python_version>="3.6" 50 | invoke 51 | parver 52 | sphinx 53 | sphinx-rtd-theme 54 | twine 55 | tests = 56 | hypothesis 57 | hypothesis-fspaths 58 | pytest>=5.6.10 59 | pytest-rerunfailures 60 | pytest-xdist 61 | pytest-timeout 62 | readme-renderer[md] 63 | typing = 64 | mypy;python_version>="3.4" 65 | mypy-extensions;python_version>="3.4" 66 | mypytools;python_version>="3.4" 67 | typed-ast;python_version>="3.4" 68 | 69 | [bdist_wheel] 70 | universal = 1 71 | 72 | 73 | [tool:pytest] 74 | strict = true 75 | plugins = flake8 timeout 76 | addopts = -ra --timeout 300 77 | testpaths = tests/ 78 | norecursedirs = .* build dist news tasks docs 79 | flake8-ignore = 80 | docs/source/* ALL 81 | tests/*.py ALL 82 | setup.py ALL 83 | src/vistir/backports/surrogateescape.py ALL 84 | filterwarnings = 85 | ignore::DeprecationWarning 86 | ignore::PendingDeprecationWarning 87 | 88 | [flake8] 89 | max-line-length = 90 90 | select = C,E,F,W,B,B950 91 | # select = E,W,F 92 | ignore = 93 | # The default ignore list: 94 | # E121,E123,E126,E226,E24,E704, 95 | D203,F401,E123,E203,W503,E501 96 | # Our additions: 97 | # E127: continuation line over-indented for visual indent 98 | # E128: continuation line under-indented for visual indent 99 | # E129: visually indented line with same indent as next logical line 100 | # E222: multiple spaces after operator 101 | # E231: missing whitespace after ',' 102 | # E402: module level import not at top of file 103 | # E501: line too long 104 | # E231,E402,E501 105 | exclude = 106 | .tox, 107 | .git, 108 | __pycache__, 109 | docs/source/*, 110 | src/vistir/backports/surrogateescape.py, 111 | build, 112 | dist, 113 | tests/*, 114 | *.pyc, 115 | *.egg-info, 116 | .cache, 117 | .eggs, 118 | setup.py, 119 | max-complexity=13 120 | 121 | [isort] 122 | atomic = true 123 | not_skip = __init__.py 124 | line_length = 90 125 | indent = ' ' 126 | multi_line_output = 3 127 | known_third_party = colorama,hypothesis,hypothesis_fspaths,invoke,msvcrt,parver,pytest,setuptools,six,towncrier 128 | known_first_party = vistir,tests 129 | combine_as_imports=True 130 | include_trailing_comma = True 131 | force_grid_wrap=0 132 | 133 | [mypy] 134 | ignore_missing_imports=true 135 | follow_imports=skip 136 | html_report=mypyhtml 137 | python_version=2.7 138 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | 4 | from setuptools import find_packages, setup 5 | 6 | 7 | ROOT = os.path.dirname(__file__) 8 | 9 | PACKAGE_NAME = 'vistir' 10 | 11 | VERSION = None 12 | 13 | with open(os.path.join(ROOT, 'src', PACKAGE_NAME, '__init__.py')) as f: 14 | for line in f: 15 | if line.startswith('__version__ = '): 16 | VERSION = ast.literal_eval(line[len('__version__ = '):].strip()) 17 | break 18 | if VERSION is None: 19 | raise EnvironmentError('failed to read version') 20 | 21 | 22 | # Put everything in setup.cfg, except those that don't actually work? 23 | setup( 24 | # These really don't work. 25 | package_dir={'': 'src'}, 26 | packages=find_packages('src'), 27 | 28 | # I don't know how to specify an empty key in setup.cfg. 29 | package_data={ 30 | '': ['LICENSE*', 'README*'], 31 | }, 32 | 33 | # I need this to be dynamic. 34 | version=VERSION, 35 | ) 36 | -------------------------------------------------------------------------------- /src/vistir/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | 3 | __version__ = "0.8.1.dev0" 4 | -------------------------------------------------------------------------------- /src/vistir/_winconsole.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # This Module is taken in part from the click project and expanded 4 | # see https://github.com/pallets/click/blob/6cafd32/click/_winconsole.py 5 | # Copyright © 2014 by the Pallets team. 6 | 7 | # Some rights reserved. 8 | 9 | # Redistribution and use in source and binary forms of the software as well as 10 | # documentation, with or without modification, are permitted provided that the 11 | # following conditions are met: 12 | # Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the 16 | # documentation and/or other materials provided with the distribution. 17 | # Neither the name of the copyright holder nor the names of its contributors 18 | # may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | 21 | # THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND 31 | # DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # This module is based on the excellent work by Adam Bartoš who 34 | # provided a lot of what went into the implementation here in 35 | # the discussion to issue1602 in the Python bug tracker. 36 | # 37 | # There are some general differences in regards to how this works 38 | # compared to the original patches as we do not need to patch 39 | # the entire interpreter but just work in our little world of 40 | # echo and prmopt. 41 | 42 | import ctypes 43 | import io 44 | import os 45 | import sys 46 | import time 47 | import typing 48 | import zlib 49 | from ctypes import ( 50 | POINTER, 51 | WINFUNCTYPE, 52 | Structure, 53 | byref, 54 | c_char, 55 | c_char_p, 56 | c_int, 57 | c_ssize_t, 58 | c_ulong, 59 | c_void_p, 60 | create_unicode_buffer, 61 | py_object, 62 | windll, 63 | ) 64 | from ctypes.wintypes import HANDLE, LPCWSTR, LPWSTR 65 | from itertools import count 66 | 67 | import msvcrt 68 | 69 | from .misc import StreamWrapper, run, to_text 70 | 71 | try: 72 | from ctypes import pythonapi 73 | 74 | PyObject_GetBuffer = pythonapi.PyObject_GetBuffer 75 | PyBuffer_Release = pythonapi.PyBuffer_Release 76 | except ImportError: 77 | pythonapi = None 78 | 79 | 80 | if typing.TYPE_CHECKING: 81 | from typing import Text 82 | 83 | 84 | c_ssize_p = POINTER(c_ssize_t) 85 | CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( 86 | ("CommandLineToArgvW", windll.shell32) 87 | ) 88 | kernel32 = windll.kernel32 89 | GetLastError = kernel32.GetLastError 90 | GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) 91 | GetConsoleCursorInfo = kernel32.GetConsoleCursorInfo 92 | GetStdHandle = kernel32.GetStdHandle 93 | LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("LocalFree", windll.kernel32)) 94 | ReadConsoleW = kernel32.ReadConsoleW 95 | SetConsoleCursorInfo = kernel32.SetConsoleCursorInfo 96 | WriteConsoleW = kernel32.WriteConsoleW 97 | 98 | # XXX: Added for cursor hiding on windows 99 | STDOUT_HANDLE_ID = ctypes.c_ulong(-11) 100 | STDERR_HANDLE_ID = ctypes.c_ulong(-12) 101 | STDIN_HANDLE = GetStdHandle(-10) 102 | STDOUT_HANDLE = GetStdHandle(-11) 103 | STDERR_HANDLE = GetStdHandle(-12) 104 | 105 | STREAM_MAP = {0: STDIN_HANDLE, 1: STDOUT_HANDLE, 2: STDERR_HANDLE} 106 | 107 | 108 | PyBUF_SIMPLE = 0 109 | PyBUF_WRITABLE = 1 110 | 111 | ERROR_SUCCESS = 0 112 | ERROR_NOT_ENOUGH_MEMORY = 8 113 | ERROR_OPERATION_ABORTED = 995 114 | 115 | STDIN_FILENO = 0 116 | STDOUT_FILENO = 1 117 | STDERR_FILENO = 2 118 | 119 | EOF = b"\x1a" 120 | MAX_BYTES_WRITTEN = 32767 121 | 122 | 123 | class Py_buffer(Structure): 124 | _fields_ = [ 125 | ("buf", c_void_p), 126 | ("obj", py_object), 127 | ("len", c_ssize_t), 128 | ("itemsize", c_ssize_t), 129 | ("readonly", c_int), 130 | ("ndim", c_int), 131 | ("format", c_char_p), 132 | ("shape", c_ssize_p), 133 | ("strides", c_ssize_p), 134 | ("suboffsets", c_ssize_p), 135 | ("internal", c_void_p), 136 | ] 137 | 138 | 139 | # XXX: This was added for the use of cursors 140 | class CONSOLE_CURSOR_INFO(Structure): 141 | _fields_ = [("dwSize", ctypes.c_int), ("bVisible", ctypes.c_int)] 142 | 143 | 144 | # On PyPy we cannot get buffers so our ability to operate here is 145 | # serverly limited. 146 | if pythonapi is None: 147 | get_buffer = None 148 | else: 149 | 150 | def get_buffer(obj, writable=False): 151 | buf = Py_buffer() 152 | flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE 153 | PyObject_GetBuffer(py_object(obj), byref(buf), flags) 154 | try: 155 | buffer_type = c_char * buf.len 156 | return buffer_type.from_address(buf.buf) 157 | finally: 158 | PyBuffer_Release(byref(buf)) 159 | 160 | 161 | def get_long_path(short_path): 162 | # type: (Text, str) -> Text 163 | BUFFER_SIZE = 500 164 | buffer = create_unicode_buffer(BUFFER_SIZE) 165 | get_long_path_name = windll.kernel32.GetLongPathNameW 166 | get_long_path_name(to_text(short_path), buffer, BUFFER_SIZE) 167 | return buffer.value 168 | 169 | 170 | class _WindowsConsoleRawIOBase(io.RawIOBase): 171 | def __init__(self, handle): 172 | self.handle = handle 173 | 174 | def isatty(self): 175 | io.RawIOBase.isatty(self) 176 | return True 177 | 178 | 179 | class _WindowsConsoleReader(_WindowsConsoleRawIOBase): 180 | def readable(self): 181 | return True 182 | 183 | def readinto(self, b): 184 | bytes_to_be_read = len(b) 185 | if not bytes_to_be_read: 186 | return 0 187 | elif bytes_to_be_read % 2: 188 | raise ValueError( 189 | "cannot read odd number of bytes from " "UTF-16-LE encoded console" 190 | ) 191 | 192 | buffer = get_buffer(b, writable=True) 193 | code_units_to_be_read = bytes_to_be_read // 2 194 | code_units_read = c_ulong() 195 | 196 | rv = ReadConsoleW( 197 | self.handle, buffer, code_units_to_be_read, byref(code_units_read), None 198 | ) 199 | if GetLastError() == ERROR_OPERATION_ABORTED: 200 | # wait for KeyboardInterrupt 201 | time.sleep(0.1) 202 | if not rv: 203 | raise OSError("Windows error: %s" % GetLastError()) 204 | 205 | if buffer[0] == EOF: 206 | return 0 207 | return 2 * code_units_read.value 208 | 209 | 210 | class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): 211 | def writable(self): 212 | return True 213 | 214 | @staticmethod 215 | def _get_error_message(errno): 216 | if errno == ERROR_SUCCESS: 217 | return "ERROR_SUCCESS" 218 | elif errno == ERROR_NOT_ENOUGH_MEMORY: 219 | return "ERROR_NOT_ENOUGH_MEMORY" 220 | return "Windows error %s" % errno 221 | 222 | def write(self, b): 223 | bytes_to_be_written = len(b) 224 | buf = get_buffer(b) 225 | code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 226 | code_units_written = c_ulong() 227 | 228 | WriteConsoleW( 229 | self.handle, buf, code_units_to_be_written, byref(code_units_written), None 230 | ) 231 | bytes_written = 2 * code_units_written.value 232 | 233 | if bytes_written == 0 and bytes_to_be_written > 0: 234 | raise OSError(self._get_error_message(GetLastError())) 235 | return bytes_written 236 | 237 | 238 | class ConsoleStream(object): 239 | def __init__(self, text_stream, byte_stream): 240 | self._text_stream = text_stream 241 | self.buffer = byte_stream 242 | 243 | @property 244 | def name(self): 245 | return self.buffer.name 246 | 247 | @property 248 | def fileno(self): 249 | return self.buffer.fileno 250 | 251 | def write(self, x): 252 | if isinstance(x, str): 253 | return self._text_stream.write(x) 254 | try: 255 | self.flush() 256 | except Exception: 257 | pass 258 | return self.buffer.write(x) 259 | 260 | def writelines(self, lines): 261 | for line in lines: 262 | self.write(line) 263 | 264 | def __getattr__(self, name): 265 | try: 266 | return getattr(self._text_stream, name) 267 | except io.UnsupportedOperation: 268 | return getattr(self.buffer, name) 269 | 270 | def isatty(self): 271 | return self.buffer.isatty() 272 | 273 | def __repr__(self): 274 | return "" % (self.name, self.encoding) 275 | 276 | 277 | class WindowsChunkedWriter(object): 278 | """ 279 | Wraps a stream (such as stdout), acting as a transparent proxy for all 280 | attribute access apart from method 'write()' which we wrap to write in 281 | limited chunks due to a Windows limitation on binary console streams. 282 | """ 283 | 284 | def __init__(self, wrapped): 285 | # double-underscore everything to prevent clashes with names of 286 | # attributes on the wrapped stream object. 287 | self.__wrapped = wrapped 288 | 289 | def __getattr__(self, name): 290 | return getattr(self.__wrapped, name) 291 | 292 | def write(self, text): 293 | total_to_write = len(text) 294 | written = 0 295 | 296 | while written < total_to_write: 297 | to_write = min(total_to_write - written, MAX_BYTES_WRITTEN) 298 | self.__wrapped.write(text[written : written + to_write]) 299 | written += to_write 300 | 301 | 302 | _wrapped_std_streams = set() 303 | 304 | 305 | def _get_text_stdin(buffer_stream): 306 | text_stream = StreamWrapper( 307 | io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), 308 | "utf-16-le", 309 | "strict", 310 | line_buffering=True, 311 | ) 312 | return ConsoleStream(text_stream, buffer_stream) 313 | 314 | 315 | def _get_text_stdout(buffer_stream): 316 | text_stream = StreamWrapper( 317 | io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), 318 | "utf-16-le", 319 | "strict", 320 | line_buffering=True, 321 | ) 322 | return ConsoleStream(text_stream, buffer_stream) 323 | 324 | 325 | def _get_text_stderr(buffer_stream): 326 | text_stream = StreamWrapper( 327 | io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), 328 | "utf-16-le", 329 | "strict", 330 | line_buffering=True, 331 | ) 332 | return ConsoleStream(text_stream, buffer_stream) 333 | 334 | 335 | _stream_factories = {0: _get_text_stdin, 1: _get_text_stdout, 2: _get_text_stderr} 336 | 337 | 338 | def _get_windows_console_stream(f, encoding, errors): 339 | if ( 340 | get_buffer is not None 341 | and encoding in ("utf-16-le", None) 342 | and errors in ("strict", None) 343 | and hasattr(f, "isatty") 344 | and f.isatty() 345 | ): 346 | if isinstance(f, ConsoleStream): 347 | return f 348 | func = _stream_factories.get(f.fileno()) 349 | if func is not None: 350 | f = getattr(f, "buffer", None) 351 | if f is None: 352 | return None 353 | else: 354 | # If we are on Python 2 we need to set the stream that we 355 | # deal with to binary mode as otherwise the exercise if a 356 | # bit moot. The same problems apply as for 357 | # get_binary_stdin and friends from _compat. 358 | msvcrt.setmode(f.fileno(), os.O_BINARY) 359 | return func(f) 360 | 361 | 362 | def hide_cursor(): 363 | cursor_info = CONSOLE_CURSOR_INFO() 364 | GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) 365 | cursor_info.visible = False 366 | SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) 367 | 368 | 369 | def show_cursor(): 370 | cursor_info = CONSOLE_CURSOR_INFO() 371 | GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) 372 | cursor_info.visible = True 373 | SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) 374 | 375 | 376 | def get_stream_handle(stream): 377 | return STREAM_MAP.get(stream.fileno()) 378 | 379 | 380 | def _walk_for_powershell(directory): 381 | for path, dirs, files in os.walk(directory): 382 | powershell = next( 383 | iter(fn for fn in files if fn.lower() == "powershell.exe"), None 384 | ) 385 | if powershell is not None: 386 | return os.path.join(directory, powershell) 387 | for subdir in dirs: 388 | powershell = _walk_for_powershell(os.path.join(directory, subdir)) 389 | if powershell: 390 | return powershell 391 | return None 392 | 393 | 394 | def _get_powershell_path(): 395 | paths = [ 396 | os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir) 397 | for subdir in ("SysWOW64", "system32") 398 | ] 399 | powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None) 400 | if not powershell_path: 401 | powershell_path, _ = run( 402 | ["where", "powershell"], block=True, nospin=True, return_object=False 403 | ) 404 | if powershell_path: 405 | return powershell_path.strip() 406 | return None 407 | 408 | 409 | def _get_sid_with_powershell(): 410 | powershell_path = _get_powershell_path() 411 | if not powershell_path: 412 | return None 413 | args = [ 414 | powershell_path, 415 | "-ExecutionPolicy", 416 | "Bypass", 417 | "-Command", 418 | "Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'", 419 | ] 420 | sid, _ = run(args, nospin=True) 421 | return sid.strip() 422 | 423 | 424 | def _get_sid_from_registry(): 425 | try: 426 | import winreg 427 | except ImportError: 428 | import _winreg as winreg 429 | var_names = ("%USERPROFILE%", "%HOME%") 430 | current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None) 431 | root, subkey = ( 432 | winreg.HKEY_LOCAL_MACHINE, 433 | r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList", 434 | ) 435 | subkey_names = [] 436 | value = None 437 | matching_key = None 438 | try: 439 | with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key: 440 | for i in count(): 441 | key_name = winreg.EnumKey(key, i) 442 | subkey_names.append(key_name) 443 | value = query_registry_value( 444 | root, r"{0}\{1}".format(subkey, key_name), "ProfileImagePath" 445 | ) 446 | if value and value.lower() == current_user_home.lower(): 447 | matching_key = key_name 448 | break 449 | except OSError: 450 | pass 451 | if matching_key is not None: 452 | return matching_key 453 | 454 | 455 | def get_value_from_tuple(value, value_type): 456 | try: 457 | import winreg 458 | except ImportError: 459 | import _winreg as winreg 460 | if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ): 461 | if "\0" in value: 462 | return value[: value.index("\0")] 463 | return value 464 | return None 465 | 466 | 467 | def query_registry_value(root, key_name, value): 468 | try: 469 | import winreg 470 | except ImportError: 471 | import _winreg as winreg 472 | try: 473 | with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key: 474 | return get_value_from_tuple(*winreg.QueryValueEx(key, value)) 475 | except OSError: 476 | return None 477 | 478 | 479 | def get_current_user(): 480 | fns = (_get_sid_from_registry, _get_sid_with_powershell) 481 | for fn in fns: 482 | result = fn() 483 | if result: 484 | return result 485 | return None 486 | -------------------------------------------------------------------------------- /src/vistir/cmdparse.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import itertools 5 | import re 6 | import shlex 7 | 8 | __all__ = ["ScriptEmptyError", "Script"] 9 | 10 | 11 | class ScriptEmptyError(ValueError): 12 | pass 13 | 14 | 15 | def _quote_if_contains(value, pattern): 16 | if next(re.finditer(pattern, value), None): 17 | return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) 18 | return value 19 | 20 | 21 | class Script(object): 22 | """Parse a script line (in Pipfile's [scripts] section). 23 | 24 | This always works in POSIX mode, even on Windows. 25 | """ 26 | 27 | def __init__(self, command, args=None): 28 | self._parts = [command] 29 | if args: 30 | self._parts.extend(args) 31 | 32 | @classmethod 33 | def parse(cls, value): 34 | if isinstance(value, str): 35 | value = shlex.split(value) 36 | if not value: 37 | raise ScriptEmptyError(value) 38 | return cls(value[0], value[1:]) 39 | 40 | def __repr__(self): 41 | return "Script({0!r})".format(self._parts) 42 | 43 | @property 44 | def command(self): 45 | return self._parts[0] 46 | 47 | @property 48 | def args(self): 49 | return self._parts[1:] 50 | 51 | def extend(self, extra_args): 52 | self._parts.extend(extra_args) 53 | 54 | def cmdify(self): 55 | """Encode into a cmd-executable string. 56 | 57 | This re-implements CreateProcess's quoting logic to turn a list of 58 | arguments into one single string for the shell to interpret. 59 | 60 | * All double quotes are escaped with a backslash. 61 | * Existing backslashes before a quote are doubled, so they are all 62 | escaped properly. 63 | * Backslashes elsewhere are left as-is; cmd will interpret them 64 | literally. 65 | 66 | The result is then quoted into a pair of double quotes to be grouped. 67 | 68 | An argument is intentionally not quoted if it does not contain 69 | whitespaces. This is done to be compatible with Windows built-in 70 | commands that don't work well with quotes, e.g. everything with `echo`, 71 | and DOS-style (forward slash) switches. 72 | 73 | The intended use of this function is to pre-process an argument list 74 | before passing it into ``subprocess.Popen(..., shell=True)``. 75 | 76 | See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence 77 | """ 78 | return " ".join( 79 | itertools.chain( 80 | [_quote_if_contains(self.command, r"[\s^()]")], 81 | (_quote_if_contains(arg, r"[\s^]") for arg in self.args), 82 | ) 83 | ) 84 | -------------------------------------------------------------------------------- /src/vistir/contextmanagers.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | import io 3 | import os 4 | 5 | import stat 6 | import sys 7 | import typing 8 | 9 | from contextlib import closing, contextmanager 10 | from pathlib import Path 11 | from tempfile import NamedTemporaryFile 12 | from urllib import request 13 | 14 | from .path import is_file_url, is_valid_url, path_to_url, url_to_path 15 | 16 | if typing.TYPE_CHECKING: 17 | from typing import ( 18 | Any, 19 | Bytes, 20 | Callable, 21 | ContextManager, 22 | Dict, 23 | IO, 24 | Iterator, 25 | Optional, 26 | Union, 27 | Text, 28 | Tuple, 29 | TypeVar, 30 | ) 31 | from types import ModuleType 32 | from requests import Session 33 | from http.client import HTTPResponse as Urllib_HTTPResponse 34 | from urllib3.response import HTTPResponse as Urllib3_HTTPResponse 35 | from .spin import VistirSpinner, DummySpinner 36 | 37 | TSpinner = Union[VistirSpinner, DummySpinner] 38 | _T = TypeVar("_T") 39 | 40 | 41 | __all__ = [ 42 | "temp_environ", 43 | "temp_path", 44 | "cd", 45 | "atomic_open_for_write", 46 | "open_file", 47 | "replaced_stream", 48 | "replaced_streams", 49 | ] 50 | 51 | 52 | # Borrowed from Pew. 53 | # See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 54 | @contextmanager 55 | def temp_environ(): 56 | # type: () -> Iterator[None] 57 | """Allow the ability to set os.environ temporarily""" 58 | environ = dict(os.environ) 59 | try: 60 | yield 61 | finally: 62 | os.environ.clear() 63 | os.environ.update(environ) 64 | 65 | 66 | @contextmanager 67 | def temp_path(): 68 | # type: () -> Iterator[None] 69 | """A context manager which allows the ability to set sys.path temporarily 70 | 71 | >>> path_from_virtualenv = load_path("/path/to/venv/bin/python") 72 | >>> print(sys.path) 73 | [ 74 | '/home/user/.pyenv/versions/3.7.0/bin', 75 | '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', 76 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7', 77 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', 78 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages' 79 | ] 80 | >>> with temp_path(): 81 | sys.path = path_from_virtualenv 82 | # Running in the context of the path above 83 | run(["pip", "install", "stuff"]) 84 | >>> print(sys.path) 85 | [ 86 | '/home/user/.pyenv/versions/3.7.0/bin', 87 | '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', 88 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7', 89 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', 90 | '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages' 91 | ] 92 | 93 | """ 94 | path = [p for p in sys.path] 95 | try: 96 | yield 97 | finally: 98 | sys.path = [p for p in path] 99 | 100 | 101 | @contextmanager 102 | def cd(path): 103 | # type: () -> Iterator[None] 104 | """Context manager to temporarily change working directories 105 | 106 | :param str path: The directory to move into 107 | 108 | >>> print(os.path.abspath(os.curdir)) 109 | '/home/user/code/myrepo' 110 | >>> with cd("/home/user/code/otherdir/subdir"): 111 | ... print("Changed directory: %s" % os.path.abspath(os.curdir)) 112 | Changed directory: /home/user/code/otherdir/subdir 113 | >>> print(os.path.abspath(os.curdir)) 114 | '/home/user/code/myrepo' 115 | """ 116 | if not path: 117 | return 118 | prev_cwd = Path.cwd().as_posix() 119 | if isinstance(path, Path): 120 | path = path.as_posix() 121 | os.chdir(str(path)) 122 | try: 123 | yield 124 | finally: 125 | os.chdir(prev_cwd) 126 | 127 | 128 | @contextmanager 129 | def atomic_open_for_write(target, binary=False, newline=None, encoding=None): 130 | # type: (str, bool, Optional[str], Optional[str]) -> None 131 | """Atomically open `target` for writing. 132 | 133 | This is based on Lektor's `atomic_open()` utility, but simplified a lot 134 | to handle only writing, and skip many multi-process/thread edge cases 135 | handled by Werkzeug. 136 | 137 | :param str target: Target filename to write 138 | :param bool binary: Whether to open in binary mode, default False 139 | :param Optional[str] newline: The newline character to use when writing, determined 140 | from system if not supplied. 141 | :param Optional[str] encoding: The encoding to use when writing, defaults to system 142 | encoding. 143 | 144 | How this works: 145 | 146 | * Create a temp file (in the same directory of the actual target), and 147 | yield for surrounding code to write to it. 148 | * If some thing goes wrong, try to remove the temp file. The actual target 149 | is not touched whatsoever. 150 | * If everything goes well, close the temp file, and replace the actual 151 | target with this new file. 152 | 153 | .. code:: python 154 | 155 | >>> fn = "test_file.txt" 156 | >>> def read_test_file(filename=fn): 157 | with open(filename, 'r') as fh: 158 | print(fh.read().strip()) 159 | 160 | >>> with open(fn, "w") as fh: 161 | fh.write("this is some test text") 162 | >>> read_test_file() 163 | this is some test text 164 | 165 | >>> def raise_exception_while_writing(filename): 166 | with open(filename, "w") as fh: 167 | fh.write("writing some new text") 168 | raise RuntimeError("Uh oh, hope your file didn't get overwritten") 169 | 170 | >>> raise_exception_while_writing(fn) 171 | Traceback (most recent call last): 172 | ... 173 | RuntimeError: Uh oh, hope your file didn't get overwritten 174 | >>> read_test_file() 175 | writing some new text 176 | 177 | # Now try with vistir 178 | >>> def raise_exception_while_writing(filename): 179 | with vistir.contextmanagers.atomic_open_for_write(filename) as fh: 180 | fh.write("Overwriting all the text from before with even newer text") 181 | raise RuntimeError("But did it get overwritten now?") 182 | 183 | >>> raise_exception_while_writing(fn) 184 | Traceback (most recent call last): 185 | ... 186 | RuntimeError: But did it get overwritten now? 187 | 188 | >>> read_test_file() 189 | writing some new text 190 | """ 191 | 192 | mode = "w+b" if binary else "w" 193 | f = NamedTemporaryFile( 194 | dir=os.path.dirname(target), 195 | prefix=".__atomic-write", 196 | mode=mode, 197 | encoding=encoding, 198 | newline=newline, 199 | delete=False, 200 | ) 201 | # set permissions to 0644 202 | try: 203 | os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) 204 | except OSError: 205 | pass 206 | try: 207 | yield f 208 | except BaseException: 209 | f.close() 210 | try: 211 | os.remove(f.name) 212 | except OSError: 213 | pass 214 | raise 215 | else: 216 | f.close() 217 | try: 218 | os.remove(target) # This is needed on Windows. 219 | except OSError: 220 | pass 221 | os.rename(f.name, target) # No os.replace() on Python 2. 222 | 223 | 224 | @contextmanager 225 | def open_file( 226 | link, # type: Union[_T, str] 227 | session=None, # type: Optional[Session] 228 | stream=True, # type: bool 229 | ): 230 | # type: (...) -> ContextManager[Union[IO[bytes], Urllib3_HTTPResponse, Urllib_HTTPResponse]] 231 | """ 232 | Open local or remote file for reading. 233 | 234 | :param pip._internal.index.Link link: A link object from resolving dependencies with 235 | pip, or else a URL. 236 | :param Optional[Session] session: A :class:`~requests.Session` instance 237 | :param bool stream: Whether to stream the content if remote, default True 238 | :raises ValueError: If link points to a local directory. 239 | :return: a context manager to the opened file-like object 240 | """ 241 | if not isinstance(link, str): 242 | try: 243 | link = link.url_without_fragment 244 | except AttributeError: 245 | raise ValueError("Cannot parse url from unknown type: {0!r}".format(link)) 246 | 247 | if not is_valid_url(link) and os.path.exists(link): 248 | link = path_to_url(link) 249 | 250 | if is_file_url(link): 251 | # Local URL 252 | local_path = url_to_path(link) 253 | if os.path.isdir(local_path): 254 | raise ValueError("Cannot open directory for read: {}".format(link)) 255 | else: 256 | with io.open(local_path, "rb") as local_file: 257 | yield local_file 258 | else: 259 | # Remote URL 260 | headers = {"Accept-Encoding": "identity"} 261 | if not session: 262 | try: 263 | from requests import Session # noqa 264 | except ImportError: 265 | session = None 266 | else: 267 | session = Session() 268 | if session is None: 269 | with closing(request.urlopen(link)) as f: 270 | yield f 271 | else: 272 | with session.get(link, headers=headers, stream=stream) as resp: 273 | try: 274 | raw = getattr(resp, "raw", None) 275 | result = raw if raw else resp 276 | yield result 277 | finally: 278 | if raw: 279 | conn = raw._connection 280 | if conn is not None: 281 | conn.close() 282 | result.close() 283 | 284 | 285 | @contextmanager 286 | def replaced_stream(stream_name): 287 | # type: (str) -> Iterator[IO[Text]] 288 | """ 289 | Context manager to temporarily swap out *stream_name* with a stream wrapper. 290 | 291 | :param str stream_name: The name of a sys stream to wrap 292 | :returns: A ``StreamWrapper`` replacement, temporarily 293 | 294 | >>> orig_stdout = sys.stdout 295 | >>> with replaced_stream("stdout") as stdout: 296 | ... sys.stdout.write("hello") 297 | ... assert stdout.getvalue() == "hello" 298 | 299 | >>> sys.stdout.write("hello") 300 | 'hello' 301 | """ 302 | 303 | orig_stream = getattr(sys, stream_name) 304 | new_stream = io.StringIO() 305 | try: 306 | setattr(sys, stream_name, new_stream) 307 | yield getattr(sys, stream_name) 308 | finally: 309 | setattr(sys, stream_name, orig_stream) 310 | 311 | 312 | @contextmanager 313 | def replaced_streams(): 314 | # type: () -> Iterator[Tuple[IO[Text], IO[Text]]] 315 | """ 316 | Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using 317 | ``replaced_stream`` 318 | 319 | returns: *(stdout, stderr)* 320 | 321 | >>> import sys 322 | >>> with vistir.contextmanagers.replaced_streams() as streams: 323 | >>> stdout, stderr = streams 324 | >>> sys.stderr.write("test") 325 | >>> sys.stdout.write("hello") 326 | >>> assert stdout.getvalue() == "hello" 327 | >>> assert stderr.getvalue() == "test" 328 | 329 | >>> stdout.getvalue() 330 | 'hello' 331 | 332 | >>> stderr.getvalue() 333 | 'test' 334 | """ 335 | 336 | with replaced_stream("stdout") as stdout: 337 | with replaced_stream("stderr") as stderr: 338 | yield (stdout, stderr) 339 | -------------------------------------------------------------------------------- /src/vistir/cursor.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, print_function 3 | 4 | import os 5 | import sys 6 | 7 | __all__ = ["hide_cursor", "show_cursor", "get_stream_handle"] 8 | 9 | 10 | def get_stream_handle(stream=sys.stdout): 11 | """ 12 | Get the OS appropriate handle for the corresponding output stream. 13 | 14 | :param str stream: The the stream to get the handle for 15 | :return: A handle to the appropriate stream, either a ctypes buffer 16 | or **sys.stdout** or **sys.stderr**. 17 | """ 18 | handle = stream 19 | if os.name == "nt": 20 | from ._winconsole import get_stream_handle as get_win_stream_handle 21 | 22 | return get_win_stream_handle(stream) 23 | return handle 24 | 25 | 26 | def hide_cursor(stream=sys.stdout): 27 | """ 28 | Hide the console cursor on the given stream 29 | 30 | :param stream: The name of the stream to get the handle for 31 | :return: None 32 | :rtype: None 33 | """ 34 | 35 | handle = get_stream_handle(stream=stream) 36 | if os.name == "nt": 37 | from ._winconsole import hide_cursor 38 | 39 | hide_cursor() 40 | else: 41 | handle.write("\033[?25l") 42 | handle.flush() 43 | 44 | 45 | def show_cursor(stream=sys.stdout): 46 | """ 47 | Show the console cursor on the given stream 48 | 49 | :param stream: The name of the stream to get the handle for 50 | :return: None 51 | :rtype: None 52 | """ 53 | 54 | handle = get_stream_handle(stream=stream) 55 | if os.name == "nt": 56 | from ._winconsole import show_cursor 57 | 58 | show_cursor() 59 | else: 60 | handle.write("\033[?25h") 61 | handle.flush() 62 | -------------------------------------------------------------------------------- /src/vistir/path.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, print_function, unicode_literals 3 | 4 | import atexit 5 | import errno 6 | import functools 7 | import locale 8 | import os 9 | import posixpath 10 | import shutil 11 | import stat 12 | import sys 13 | import typing 14 | import time 15 | import unicodedata 16 | import warnings 17 | 18 | from pathlib import Path 19 | from tempfile import NamedTemporaryFile, TemporaryDirectory 20 | from typing import Optional, Callable 21 | from urllib import parse as urllib_parse 22 | from urllib import request as urllib_request 23 | 24 | from urllib.parse import quote 25 | 26 | if typing.TYPE_CHECKING: 27 | from types import TracebackType 28 | from typing import ( 29 | Any, 30 | AnyStr, 31 | ByteString, 32 | Generator, 33 | Iterator, 34 | List, 35 | Text, 36 | Tuple, 37 | Type, 38 | Union, 39 | ) 40 | 41 | TPath = os.PathLike 42 | TFunc = Callable[..., Any] 43 | 44 | 45 | __all__ = [ 46 | "check_for_unc_path", 47 | "get_converted_relative_path", 48 | "handle_remove_readonly", 49 | "normalize_path", 50 | "is_in_path", 51 | "is_file_url", 52 | "is_readonly_path", 53 | "is_valid_url", 54 | "mkdir_p", 55 | "ensure_mkdir_p", 56 | "create_tracked_tempdir", 57 | "create_tracked_tempfile", 58 | "path_to_url", 59 | "rmtree", 60 | "safe_expandvars", 61 | "set_write_bit", 62 | "url_to_path", 63 | "walk_up", 64 | ] 65 | 66 | 67 | if os.name == "nt": 68 | warnings.filterwarnings( 69 | "ignore", 70 | category=DeprecationWarning, 71 | message="The Windows bytes API has been deprecated.*", 72 | ) 73 | 74 | 75 | def unicode_path(path): 76 | # type: (TPath) -> Text 77 | # Paths are supposed to be represented as unicode here 78 | return path 79 | 80 | 81 | def native_path(path): 82 | # type: (TPath) -> str 83 | return str(path) 84 | 85 | 86 | # once again thank you django... 87 | # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py 88 | if os.name == "nt": 89 | abspathu = os.path.abspath 90 | else: 91 | 92 | def abspathu(path): 93 | # type: (TPath) -> Text 94 | """Version of os.path.abspath that uses the unicode representation of 95 | the current working directory, thus avoiding a UnicodeDecodeError in 96 | join when the cwd has non-ASCII characters.""" 97 | if not os.path.isabs(path): 98 | path = os.path.join(os.getcwd(), path) 99 | return os.path.normpath(path) 100 | 101 | 102 | def normalize_path(path): 103 | # type: (TPath) -> Text 104 | """Return a case-normalized absolute variable-expanded path. 105 | 106 | :param str path: The non-normalized path 107 | :return: A normalized, expanded, case-normalized path 108 | :rtype: str 109 | """ 110 | 111 | path = os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) 112 | if os.name == "nt" and os.path.exists(path): 113 | from ._winconsole import get_long_path 114 | 115 | path = get_long_path(path) 116 | 117 | return os.path.normpath(os.path.normcase(path)) 118 | 119 | 120 | def is_in_path(path, parent): 121 | # type: (TPath, TPath) -> bool 122 | """Determine if the provided full path is in the given parent root. 123 | 124 | :param str path: The full path to check the location of. 125 | :param str parent: The parent path to check for membership in 126 | :return: Whether the full path is a member of the provided parent. 127 | :rtype: bool 128 | """ 129 | 130 | return normalize_path(path).startswith(normalize_path(parent)) 131 | 132 | 133 | def normalize_drive(path): 134 | # type: (TPath) -> Text 135 | """Normalize drive in path so they stay consistent. 136 | 137 | This currently only affects local drives on Windows, which can be 138 | identified with either upper or lower cased drive names. The case is 139 | always converted to uppercase because it seems to be preferred. 140 | """ 141 | from .misc import to_text 142 | 143 | if os.name != "nt" or not ( 144 | isinstance(path, str) or getattr(path, "__fspath__", None) 145 | ): 146 | return path # type: ignore 147 | 148 | drive, tail = os.path.splitdrive(path) 149 | # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. 150 | if drive.islower() and len(drive) == 2 and drive[1] == ":": 151 | return "{}{}".format(drive.upper(), tail) 152 | 153 | return to_text(path, encoding="utf-8") 154 | 155 | 156 | def path_to_url(path): 157 | # type: (TPath) -> Text 158 | """Convert the supplied local path to a file uri. 159 | 160 | :param str path: A string pointing to or representing a local path 161 | :return: A `file://` uri for the same location 162 | :rtype: str 163 | 164 | >>> path_to_url("/home/user/code/myrepo/myfile.zip") 165 | 'file:///home/user/code/myrepo/myfile.zip' 166 | """ 167 | from .misc import to_bytes 168 | 169 | if not path: 170 | return path # type: ignore 171 | normalized_path = Path(normalize_drive(os.path.abspath(path))).as_posix() 172 | if os.name == "nt" and normalized_path[1] == ":": 173 | drive, _, path = normalized_path.partition(":") 174 | # XXX: This enables us to handle half-surrogates that were never 175 | # XXX: actually part of a surrogate pair, but were just incidentally 176 | # XXX: passed in as a piece of a filename 177 | quoted_path = quote(path, errors="backslashreplace") 178 | return "file:///{}:{}".format(drive, quoted_path) 179 | # XXX: This is also here to help deal with incidental dangling surrogates 180 | # XXX: on linux, by making sure they are preserved during encoding so that 181 | # XXX: we can urlencode the backslash correctly 182 | # bytes_path = to_bytes(normalized_path, errors="backslashreplace") 183 | return "file://{}".format(quote(path, errors="backslashreplace")) 184 | 185 | 186 | def url_to_path(url): 187 | # type: (str) -> str 188 | """Convert a valid file url to a local filesystem path. 189 | 190 | Follows logic taken from pip's equivalent function 191 | """ 192 | 193 | assert is_file_url(url), "Only file: urls can be converted to local paths" 194 | _, netloc, path, _, _ = urllib_parse.urlsplit(url) 195 | # Netlocs are UNC paths 196 | if netloc: 197 | netloc = "\\\\" + netloc 198 | 199 | path = urllib_request.url2pathname(netloc + path) 200 | return urllib_parse.unquote(path) 201 | 202 | 203 | def is_valid_url(url): 204 | # type: (Union[str, bytes]) -> bool 205 | """Checks if a given string is an url.""" 206 | from .misc import to_text 207 | 208 | if not url: 209 | return url # type: ignore 210 | pieces = urllib_parse.urlparse(to_text(url)) 211 | return all([pieces.scheme, pieces.netloc]) 212 | 213 | 214 | def is_file_url(url): 215 | # type: (Any) -> bool 216 | """Returns true if the given url is a file url.""" 217 | from .misc import to_text 218 | 219 | if not url: 220 | return False 221 | if not isinstance(url, str): 222 | try: 223 | url = url.url 224 | except AttributeError: 225 | raise ValueError("Cannot parse url from unknown type: {!r}".format(url)) 226 | url = to_text(url, encoding="utf-8") 227 | return urllib_parse.urlparse(url.lower()).scheme == "file" 228 | 229 | 230 | def is_readonly_path(fn): 231 | # type: (TPath) -> bool 232 | """Check if a provided path exists and is readonly. 233 | 234 | Permissions check is `bool(path.stat & stat.S_IREAD)` or `not 235 | os.access(path, os.W_OK)` 236 | """ 237 | if os.path.exists(fn): 238 | file_stat = os.stat(fn).st_mode 239 | return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) 240 | return False 241 | 242 | 243 | def mkdir_p(newdir, mode=0o777): 244 | warnings.warn( 245 | ('This function is deprecated and will be removed in version 0.8.' 246 | 'Use os.makedirs instead'), DeprecationWarning, stacklevel=2) 247 | # This exists in shutil already 248 | # type: (TPath, int) -> None 249 | """Recursively creates the target directory and all of its parents if they 250 | do not already exist. Fails silently if they do. 251 | 252 | :param str newdir: The directory path to ensure 253 | :raises: OSError if a file is encountered along the way 254 | """ 255 | if os.path.exists(newdir): 256 | if not os.path.isdir(newdir): 257 | raise OSError( 258 | "a file with the same name as the desired dir, '{}', already exists.".format( 259 | newdir 260 | ) 261 | ) 262 | return None 263 | os.makedirs(newdir, mode) 264 | 265 | 266 | def ensure_mkdir_p(mode=0o777): 267 | # type: (int) -> Callable[Callable[..., Any], Callable[..., Any]] 268 | """Decorator to ensure `mkdir_p` is called to the function's return 269 | value.""" 270 | warnings.warn('This function is deprecated and will be removed in version 0.8.', 271 | DeprecationWarning, stacklevel=2) 272 | 273 | # This exists in shutil already 274 | def decorator(f): 275 | # type: (Callable[..., Any]) -> Callable[..., Any] 276 | @functools.wraps(f) 277 | def decorated(*args, **kwargs): 278 | # type: () -> str 279 | path = f(*args, **kwargs) 280 | mkdir_p(path, mode=mode) 281 | return path 282 | return decorated 283 | return decorator 284 | 285 | 286 | TRACKED_TEMPORARY_DIRECTORIES = [] 287 | 288 | 289 | def create_tracked_tempdir(*args, **kwargs): 290 | # type: (Any, Any) -> str 291 | """Create a tracked temporary directory. 292 | 293 | This uses `TemporaryDirectory`, but does not remove the directory when 294 | the return value goes out of scope, instead registers a handler to cleanup 295 | on program exit. 296 | 297 | The return value is the path to the created directory. 298 | """ 299 | 300 | tempdir = TemporaryDirectory(*args, **kwargs) 301 | TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) 302 | atexit.register(tempdir.cleanup) 303 | warnings.simplefilter("ignore", ResourceWarning) 304 | return tempdir.name 305 | 306 | 307 | def create_tracked_tempfile(*args, **kwargs): 308 | # type: (Any, Any) -> str 309 | """Create a tracked temporary file. 310 | 311 | This uses the `NamedTemporaryFile` construct, but does not remove the file 312 | until the interpreter exits. 313 | 314 | The return value is the file object. 315 | """ 316 | 317 | return NamedTemporaryFile(*args, **kwargs) 318 | 319 | 320 | def _find_icacls_exe(): 321 | # type: () -> Optional[Text] 322 | if os.name == "nt": 323 | paths = [ 324 | os.path.expandvars(r"%windir%\{0}").format(subdir) 325 | for subdir in ("system32", "SysWOW64") 326 | ] 327 | for path in paths: 328 | icacls_path = next( 329 | iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None 330 | ) 331 | if icacls_path is not None: 332 | icacls_path = os.path.join(path, icacls_path) 333 | return icacls_path 334 | return None 335 | 336 | 337 | def set_write_bit(fn: str) -> None: 338 | """Set read-write permissions for the current user on the target path. Fail 339 | silently if the path doesn't exist. 340 | 341 | :param str fn: The target filename or path 342 | :return: None 343 | """ 344 | if not os.path.exists(fn): 345 | return 346 | file_stat = os.stat(fn).st_mode 347 | os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) 348 | if os.name == "nt": 349 | from ._winconsole import get_current_user 350 | 351 | user_sid = get_current_user() 352 | icacls_exe = _find_icacls_exe() or "icacls" 353 | from .misc import run 354 | 355 | if user_sid: 356 | c = run( 357 | [ 358 | icacls_exe, 359 | "''{}''".format(fn), 360 | "/grant", 361 | "{}:WD".format(user_sid), 362 | "/T", 363 | "/C", 364 | "/Q", 365 | ], 366 | nospin=True, 367 | return_object=True, 368 | # 2020-06-12 Yukihiko Shinoda 369 | # There are 3 way to get system default encoding in Stack Overflow. 370 | # see: https://stackoverflow.com/questions/37506535/how-to-get-the-system-default-encoding-in-python-2-x 371 | # I investigated these way by using Shift-JIS Windows. 372 | # >>> import locale 373 | # >>> locale.getpreferredencoding() 374 | # "cp932" (Shift-JIS) 375 | # >>> import sys 376 | # >>> sys.getdefaultencoding() 377 | # "utf-8" 378 | # >>> sys.stdout.encoding 379 | # "UTF8" 380 | encoding=locale.getpreferredencoding(), 381 | ) 382 | if not c.err and c.returncode == 0: 383 | return 384 | 385 | if not os.path.isdir(fn): 386 | for path in [fn, os.path.dirname(fn)]: 387 | try: 388 | os.chflags(path, 0) 389 | except AttributeError: 390 | pass 391 | return None 392 | for root, dirs, files in os.walk(fn, topdown=False): 393 | for dir_ in [os.path.join(root, d) for d in dirs]: 394 | set_write_bit(dir_) 395 | for file_ in [os.path.join(root, f) for f in files]: 396 | set_write_bit(file_) 397 | 398 | 399 | def rmtree(directory: str, 400 | ignore_errors: bool = False, 401 | onerror: Optional[Callable] = None) -> None : 402 | """Stand-in for :func:`~shutil.rmtree` with additional error-handling. 403 | 404 | This version of `rmtree` handles read-only paths, especially in the case of index 405 | files written by certain source control systems. 406 | 407 | :param str directory: The target directory to remove 408 | :param bool ignore_errors: Whether to ignore errors, defaults to False 409 | :param func onerror: An error handling function, defaults to :func:`handle_remove_readonly` 410 | 411 | .. note:: 412 | 413 | Setting `ignore_errors=True` may cause this to silently fail to delete the path 414 | """ 415 | 416 | if onerror is None: 417 | onerror = handle_remove_readonly 418 | try: 419 | shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) 420 | except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # noqa:B014 421 | # Ignore removal failures where the file doesn't exist 422 | if exc.errno != errno.ENOENT: 423 | raise 424 | 425 | 426 | def _wait_for_files(path): # pragma: no cover 427 | # type: (Union[str, TPath]) -> Optional[List[TPath]] 428 | """Retry with backoff up to 1 second to delete files from a directory. 429 | 430 | :param str path: The path to crawl to delete files from 431 | :return: A list of remaining paths or None 432 | :rtype: Optional[List[str]] 433 | """ 434 | timeout = 0.001 435 | remaining = [] 436 | while timeout < 1.0: 437 | remaining = [] 438 | if os.path.isdir(path): 439 | L = os.listdir(path) 440 | for target in L: 441 | _remaining = _wait_for_files(target) 442 | if _remaining: 443 | remaining.extend(_remaining) 444 | continue 445 | try: 446 | os.unlink(path) 447 | except FileNotFoundError as e: 448 | if e.errno == errno.ENOENT: 449 | return 450 | except (OSError, IOError, PermissionError): # noqa:B014 451 | time.sleep(timeout) 452 | timeout *= 2 453 | remaining.append(path) 454 | else: 455 | return 456 | return remaining 457 | 458 | 459 | def handle_remove_readonly(func, path, exc): 460 | # type: (Callable[..., str], TPath, Tuple[Type[OSError], OSError, TracebackType]) -> None 461 | """Error handler for shutil.rmtree. 462 | 463 | Windows source repo folders are read-only by default, so this error handler 464 | attempts to set them as writeable and then proceed with deletion. 465 | 466 | :param function func: The caller function 467 | :param str path: The target path for removal 468 | :param Exception exc: The raised exception 469 | 470 | This function will call check :func:`is_readonly_path` before attempting to call 471 | :func:`set_write_bit` on the target path and try again. 472 | """ 473 | 474 | PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) 475 | default_warning_message = "Unable to remove file due to permissions restriction: {!r}" 476 | # split the initial exception out into its type, exception, and traceback 477 | exc_type, exc_exception, exc_tb = exc 478 | if is_readonly_path(path): 479 | # Apply write permission and call original function 480 | set_write_bit(path) 481 | try: 482 | func(path) 483 | except ( # noqa:B014 484 | OSError, 485 | IOError, 486 | FileNotFoundError, 487 | PermissionError, 488 | ) as e: # pragma: no cover 489 | if e.errno in PERM_ERRORS: 490 | if e.errno == errno.ENOENT: 491 | return 492 | remaining = None 493 | if os.path.isdir(path): 494 | remaining = _wait_for_files(path) 495 | if remaining: 496 | warnings.warn(default_warning_message.format(path), ResourceWarning) 497 | else: 498 | func(path, ignore_errors=True) 499 | return 500 | 501 | if exc_exception.errno in PERM_ERRORS: 502 | set_write_bit(path) 503 | remaining = _wait_for_files(path) 504 | try: 505 | func(path) 506 | except (OSError, IOError, FileNotFoundError, PermissionError) as e: # noqa:B014 507 | if e.errno in PERM_ERRORS: 508 | if e.errno != errno.ENOENT: # File still exists 509 | warnings.warn(default_warning_message.format(path), ResourceWarning) 510 | return 511 | else: 512 | raise exc_exception 513 | 514 | 515 | def walk_up(bottom): 516 | # type: (Union[TPath, str]) -> Generator[Tuple[str, List[str], List[str]], None, None] 517 | """Mimic os.walk, but walk 'up' instead of down the directory tree. 518 | 519 | From: https://gist.github.com/zdavkeos/1098474 520 | """ 521 | bottom = os.path.realpath(str(bottom)) 522 | # Get files in current dir. 523 | try: 524 | names = os.listdir(bottom) 525 | except Exception: 526 | return 527 | 528 | dirs, nondirs = [], [] 529 | for name in names: 530 | if os.path.isdir(os.path.join(bottom, name)): 531 | dirs.append(name) 532 | else: 533 | nondirs.append(name) 534 | yield bottom, dirs, nondirs 535 | 536 | new_path = os.path.realpath(os.path.join(bottom, "..")) 537 | # See if we are at the top. 538 | if new_path == bottom: 539 | return 540 | 541 | for x in walk_up(new_path): 542 | yield x 543 | 544 | 545 | def check_for_unc_path(path): 546 | # type: (Path) -> bool 547 | """Checks to see if a pathlib `Path` object is a unc path or not.""" 548 | if ( 549 | os.name == "nt" 550 | and len(path.drive) > 2 551 | and not path.drive[0].isalpha() 552 | and path.drive[1] != ":" 553 | ): 554 | return True 555 | else: 556 | return False 557 | 558 | 559 | def get_converted_relative_path(path, relative_to=None): 560 | # type: (TPath, Optional[TPath]) -> str 561 | """Convert `path` to be relative. 562 | 563 | Given a vague relative path, return the path relative to the given 564 | location. 565 | 566 | :param str path: The location of a target path 567 | :param str relative_to: The starting path to build against, optional 568 | :returns: A relative posix-style path with a leading `./` 569 | 570 | This performs additional conversion to ensure the result is of POSIX form, 571 | and starts with `./`, or is precisely `.`. 572 | 573 | >>> os.chdir('/home/user/code/myrepo/myfolder') 574 | >>> vistir.path.get_converted_relative_path('/home/user/code/file.zip') 575 | './../../file.zip' 576 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder/mysubfolder') 577 | './mysubfolder' 578 | >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') 579 | '.' 580 | """ 581 | from .misc import to_text, to_bytes # noqa 582 | 583 | if not relative_to: 584 | relative_to = os.getcwd() 585 | 586 | path = to_text(path, encoding="utf-8") 587 | relative_to = to_text(relative_to, encoding="utf-8") 588 | start_path = Path(relative_to) 589 | try: 590 | start = start_path.resolve() 591 | except OSError: 592 | start = start_path.absolute() 593 | 594 | # check if there is a drive letter or mount point 595 | # if it is a mountpoint use the original absolute path 596 | # instead of the unc path 597 | if check_for_unc_path(start): 598 | start = start_path.absolute() 599 | 600 | path = start.joinpath(path).relative_to(start) 601 | 602 | # check and see if the path that was passed into the function is a UNC path 603 | # and raise value error if it is not. 604 | if check_for_unc_path(path): 605 | raise ValueError("The path argument does not currently accept UNC paths") 606 | 607 | relpath_s = to_text(posixpath.normpath(path.as_posix())) 608 | if not (relpath_s == "." or relpath_s.startswith("./")): 609 | relpath_s = posixpath.join(".", relpath_s) 610 | return relpath_s 611 | 612 | 613 | def safe_expandvars(value): 614 | # type: (TPath) -> str 615 | """Call os.path.expandvars if value is a string, otherwise do nothing.""" 616 | if isinstance(value, str): 617 | return os.path.expandvars(value) 618 | return value # type: ignore 619 | -------------------------------------------------------------------------------- /src/vistir/spin.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | import functools 3 | import os 4 | import signal 5 | import sys 6 | import threading 7 | import time 8 | import typing 9 | 10 | from io import StringIO 11 | 12 | import colorama 13 | 14 | from .cursor import hide_cursor, show_cursor 15 | from .misc import decode_for_output, to_text 16 | from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored 17 | 18 | if typing.TYPE_CHECKING: 19 | from typing import ( 20 | Any, 21 | Callable, 22 | ContextManager, 23 | Dict, 24 | IO, 25 | Optional, 26 | Text, 27 | Type, 28 | TypeVar, 29 | Union, 30 | ) 31 | 32 | TSignalMap = Dict[ 33 | Type[signal.SIGINT], 34 | Callable[..., int, str, Union["DummySpinner", "VistirSpinner"]], 35 | ] 36 | _T = TypeVar("_T", covariant=True) 37 | 38 | try: 39 | import yaspin 40 | import yaspin.spinners 41 | import yaspin.core 42 | 43 | Spinners = yaspin.spinners.Spinners 44 | SpinBase = yaspin.core.Yaspin 45 | 46 | except ImportError: # pragma: no cover 47 | yaspin = None 48 | Spinners = None 49 | SpinBase = None 50 | 51 | if os.name == "nt": # pragma: no cover 52 | 53 | def handler(signum, frame, spinner): 54 | """Signal handler, used to gracefully shut down the ``spinner`` instance 55 | when specified signal is received by the process running the ``spinner``. 56 | 57 | ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` 58 | function for more details. 59 | """ 60 | spinner.fail() 61 | spinner.stop() 62 | 63 | 64 | else: # pragma: no cover 65 | 66 | def handler(signum, frame, spinner): 67 | """Signal handler, used to gracefully shut down the ``spinner`` instance 68 | when specified signal is received by the process running the ``spinner``. 69 | 70 | ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` 71 | function for more details. 72 | """ 73 | spinner.red.fail("✘") 74 | spinner.stop() 75 | 76 | 77 | CLEAR_LINE = chr(27) + "[K" 78 | 79 | TRANSLATION_MAP = {10004: u"OK", 10008: u"x"} 80 | 81 | 82 | decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) 83 | 84 | 85 | class DummySpinner(object): 86 | def __init__(self, text="", **kwargs): 87 | # type: (str, Any) -> None 88 | if DISABLE_COLORS: 89 | colorama.init() 90 | self.text = decode_output(text) if text else "" 91 | self.stdout = kwargs.get("stdout", sys.stdout) 92 | self.stderr = kwargs.get("stderr", sys.stderr) 93 | self.out_buff = StringIO() 94 | self.write_to_stdout = kwargs.get("write_to_stdout", False) 95 | super(DummySpinner, self).__init__() 96 | 97 | def __enter__(self): 98 | if self.text and self.text != "None": 99 | if self.write_to_stdout: 100 | self.write(self.text) 101 | return self 102 | 103 | def __exit__(self, exc_type, exc_val, tb): 104 | if exc_type: 105 | import traceback 106 | 107 | formatted_tb = traceback.format_exception(exc_type, exc_val, tb) 108 | self.write_err("".join(formatted_tb)) 109 | self._close_output_buffer() 110 | return False 111 | 112 | def __getattr__(self, k): # pragma: no cover 113 | try: 114 | retval = super(DummySpinner, self).__getattribute__(k) 115 | except AttributeError: 116 | if k in COLOR_MAP.keys() or k.upper() in COLORS: 117 | return self 118 | raise 119 | else: 120 | return retval 121 | 122 | def _close_output_buffer(self): 123 | if self.out_buff and not self.out_buff.closed: 124 | try: 125 | self.out_buff.close() 126 | except Exception: 127 | pass 128 | 129 | def fail(self, exitcode=1, text="FAIL"): 130 | # type: (int, str) -> None 131 | if text is not None and text != "None": 132 | if self.write_to_stdout: 133 | self.write(text) 134 | else: 135 | self.write_err(text) 136 | self._close_output_buffer() 137 | 138 | def ok(self, text="OK"): 139 | # type: (str) -> int 140 | if text is not None and text != "None": 141 | if self.write_to_stdout: 142 | self.write(text) 143 | else: 144 | self.write_err(text) 145 | self._close_output_buffer() 146 | return 0 147 | 148 | def hide_and_write(self, text, target=None): 149 | # type: (str, Optional[str]) -> None 150 | if not target: 151 | target = self.stdout 152 | if text is None or isinstance(text, str) and text == "None": 153 | pass 154 | target.write(decode_output(u"\r", target_stream=target)) 155 | self._hide_cursor(target=target) 156 | target.write(decode_output(u"{0}\n".format(text), target_stream=target)) 157 | target.write(CLEAR_LINE) 158 | self._show_cursor(target=target) 159 | 160 | def write(self, text=None): 161 | # type: (Optional[str]) -> None 162 | if not self.write_to_stdout: 163 | return self.write_err(text) 164 | if text is None or isinstance(text, str) and text == "None": 165 | pass 166 | if not self.stdout.closed: 167 | stdout = self.stdout 168 | else: 169 | stdout = sys.stdout 170 | stdout.write(decode_output(u"\r", target_stream=stdout)) 171 | text = to_text(text) 172 | line = decode_output(u"{0}\n".format(text), target_stream=stdout) 173 | stdout.write(line) 174 | stdout.write(CLEAR_LINE) 175 | 176 | def write_err(self, text=None): 177 | # type: (Optional[str]) -> None 178 | if text is None or isinstance(text, str) and text == "None": 179 | pass 180 | text = to_text(text) 181 | if not self.stderr.closed: 182 | stderr = self.stderr 183 | else: 184 | if sys.stderr.closed: 185 | print(text) 186 | return 187 | stderr = sys.stderr 188 | stderr.write(decode_output(u"\r", target_stream=stderr)) 189 | line = decode_output(u"{0}\n".format(text), target_stream=stderr) 190 | stderr.write(line) 191 | stderr.write(CLEAR_LINE) 192 | 193 | @staticmethod 194 | def _hide_cursor(target=None): 195 | # type: (Optional[IO]) -> None 196 | pass 197 | 198 | @staticmethod 199 | def _show_cursor(target=None): 200 | # type: (Optional[IO]) -> None 201 | pass 202 | 203 | 204 | if SpinBase is None: 205 | SpinBase = DummySpinner 206 | 207 | 208 | class VistirSpinner(SpinBase): 209 | "A spinner class for handling spinners on windows and posix." 210 | 211 | def __init__(self, *args, **kwargs): 212 | # type: (Any, Any) 213 | """ 214 | Get a spinner object or a dummy spinner to wrap a context. 215 | 216 | Keyword Arguments: 217 | :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) 218 | :param str start_text: Text to start off the spinner with (default: {None}) 219 | :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) 220 | :param bool nospin: If true, use the dummy spinner (default: {False}) 221 | :param bool write_to_stdout: Writes to stdout if true, otherwise writes to stderr (default: True) 222 | """ 223 | 224 | self.handler = handler 225 | colorama.init() 226 | sigmap = {} # type: TSignalMap 227 | if handler: 228 | sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler}) 229 | handler_map = kwargs.pop("handler_map", {}) 230 | if os.name == "nt": 231 | sigmap[signal.SIGBREAK] = handler 232 | else: 233 | sigmap[signal.SIGALRM] = handler 234 | if handler_map: 235 | sigmap.update(handler_map) 236 | spinner_name = kwargs.pop("spinner_name", "bouncingBar") 237 | start_text = kwargs.pop("start_text", None) 238 | _text = kwargs.pop("text", "Running...") 239 | kwargs["text"] = start_text if start_text is not None else _text 240 | kwargs["sigmap"] = sigmap 241 | kwargs["spinner"] = getattr(Spinners, spinner_name, "") 242 | write_to_stdout = kwargs.pop("write_to_stdout", True) 243 | self.stdout = kwargs.pop("stdout", sys.stdout) 244 | self.stderr = kwargs.pop("stderr", sys.stderr) 245 | self.out_buff = StringIO() 246 | self.write_to_stdout = write_to_stdout 247 | self.is_dummy = bool(yaspin is None) 248 | self._stop_spin = None # type: Optional[threading.Event] 249 | self._hide_spin = None # type: Optional[threading.Event] 250 | self._spin_thread = None # type: Optional[threading.Thread] 251 | super(VistirSpinner, self).__init__(*args, **kwargs) 252 | if DISABLE_COLORS: 253 | colorama.deinit() 254 | 255 | def ok(self, text=u"OK", err=False): 256 | # type: (str, bool) -> None 257 | """Set Ok (success) finalizer to a spinner.""" 258 | # Do not display spin text for ok state 259 | self._text = None 260 | 261 | _text = to_text(text) if text else u"OK" 262 | err = err or not self.write_to_stdout 263 | self._freeze(_text, err=err) 264 | 265 | def fail(self, text=u"FAIL", err=False): 266 | # type: (str, bool) -> None 267 | """Set fail finalizer to a spinner.""" 268 | # Do not display spin text for fail state 269 | self._text = None 270 | 271 | _text = text if text else u"FAIL" 272 | err = err or not self.write_to_stdout 273 | self._freeze(_text, err=err) 274 | 275 | def hide_and_write(self, text, target=None): 276 | # type: (str, Optional[str]) -> None 277 | if not target: 278 | target = self.stdout 279 | if text is None or isinstance(text, str) and text == u"None": 280 | pass 281 | target.write(decode_output(u"\r")) 282 | self._hide_cursor(target=target) 283 | target.write(decode_output(u"{0}\n".format(text))) 284 | target.write(CLEAR_LINE) 285 | self._show_cursor(target=target) 286 | 287 | def write(self, text): # pragma: no cover 288 | # type: (str) -> None 289 | if not self.write_to_stdout: 290 | return self.write_err(text) 291 | stdout = self.stdout 292 | if self.stdout.closed: 293 | stdout = sys.stdout 294 | stdout.write(decode_output(u"\r", target_stream=stdout)) 295 | stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) 296 | if text is None: 297 | text = "" 298 | text = decode_output(u"{0}\n".format(text), target_stream=stdout) 299 | stdout.write(text) 300 | self.out_buff.write(text) 301 | 302 | def write_err(self, text): # pragma: no cover 303 | # type: (str) -> None 304 | """Write error text in the terminal without breaking the spinner.""" 305 | stderr = self.stderr 306 | if self.stderr.closed: 307 | stderr = sys.stderr 308 | stderr.write(decode_output(u"\r", target_stream=stderr)) 309 | stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) 310 | if text is None: 311 | text = "" 312 | text = decode_output(u"{0}\n".format(text), target_stream=stderr) 313 | self.stderr.write(text) 314 | self.out_buff.write(decode_output(text, target_stream=self.out_buff)) 315 | 316 | def start(self): 317 | # type: () -> None 318 | if self._sigmap: 319 | self._register_signal_handlers() 320 | 321 | target = self.stdout if self.write_to_stdout else self.stderr 322 | if target.isatty(): 323 | self._hide_cursor(target=target) 324 | 325 | self._stop_spin = threading.Event() 326 | self._hide_spin = threading.Event() 327 | self._spin_thread = threading.Thread(target=self._spin) 328 | self._spin_thread.start() 329 | 330 | def stop(self): 331 | # type: () -> None 332 | if self._dfl_sigmap: 333 | # Reset registered signal handlers to default ones 334 | self._reset_signal_handlers() 335 | 336 | if self._spin_thread: 337 | self._stop_spin.set() 338 | self._spin_thread.join() 339 | 340 | target = self.stdout if self.write_to_stdout else self.stderr 341 | if target.isatty(): 342 | target.write("\r") 343 | 344 | if self.write_to_stdout: 345 | self._clear_line() 346 | else: 347 | self._clear_err() 348 | 349 | if target.isatty(): 350 | self._show_cursor(target=target) 351 | self.out_buff.close() 352 | 353 | def _freeze(self, final_text, err=False): 354 | # type: (str, bool) -> None 355 | """Stop spinner, compose last frame and 'freeze' it.""" 356 | if not final_text: 357 | final_text = "" 358 | target = self.stderr if err else self.stdout 359 | if target.closed: 360 | target = sys.stderr if err else sys.stdout 361 | text = to_text(final_text) 362 | last_frame = self._compose_out(text, mode="last") 363 | self._last_frame = decode_output(last_frame, target_stream=target) 364 | 365 | # Should be stopped here, otherwise prints after 366 | # self._freeze call will mess up the spinner 367 | self.stop() 368 | target.write(self._last_frame) 369 | 370 | def _compose_color_func(self): 371 | # type: () -> Callable[..., str] 372 | fn = functools.partial( 373 | colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs) 374 | ) 375 | return fn 376 | 377 | def _compose_out(self, frame, mode=None): 378 | # type: (str, Optional[str]) -> Text 379 | # Ensure Unicode input 380 | 381 | frame = to_text(frame) 382 | if self._text is None: 383 | self._text = u"" 384 | text = to_text(self._text) 385 | if self._color_func is not None: 386 | frame = self._color_func(frame) 387 | if self._side == "right": 388 | frame, text = text, frame 389 | # Mode 390 | frame = to_text(frame) 391 | if not mode: 392 | out = u"\r{0} {1}".format(frame, text) 393 | else: 394 | out = u"{0} {1}\n".format(frame, text) 395 | return out 396 | 397 | def _spin(self): 398 | # type: () -> None 399 | target = self.stdout if self.write_to_stdout else self.stderr 400 | clear_fn = self._clear_line if self.write_to_stdout else self._clear_err 401 | while not self._stop_spin.is_set(): 402 | 403 | if self._hide_spin.is_set(): 404 | # Wait a bit to avoid wasting cycles 405 | time.sleep(self._interval) 406 | continue 407 | 408 | # Compose output 409 | spin_phase = next(self._cycle) 410 | out = self._compose_out(spin_phase) 411 | out = decode_output(out, target) 412 | 413 | # Write 414 | target.write(out) 415 | clear_fn() 416 | target.flush() 417 | 418 | # Wait 419 | time.sleep(self._interval) 420 | target.write("\b") 421 | 422 | def _register_signal_handlers(self): 423 | # type: () -> None 424 | # SIGKILL cannot be caught or ignored, and the receiving 425 | # process cannot perform any clean-up upon receiving this 426 | # signal. 427 | try: 428 | if signal.SIGKILL in self._sigmap.keys(): 429 | raise ValueError( 430 | "Trying to set handler for SIGKILL signal. " 431 | "SIGKILL cannot be cought or ignored in POSIX systems." 432 | ) 433 | except AttributeError: 434 | pass 435 | 436 | for sig, sig_handler in self._sigmap.items(): 437 | # A handler for a particular signal, once set, remains 438 | # installed until it is explicitly reset. Store default 439 | # signal handlers for subsequent reset at cleanup phase. 440 | dfl_handler = signal.getsignal(sig) 441 | self._dfl_sigmap[sig] = dfl_handler 442 | 443 | # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid 444 | # signal handlers and are not callables. 445 | if callable(sig_handler): 446 | # ``signal.signal`` accepts handler function which is 447 | # called with two arguments: signal number and the 448 | # interrupted stack frame. ``functools.partial`` solves 449 | # the problem of passing spinner instance into the handler 450 | # function. 451 | sig_handler = functools.partial(sig_handler, spinner=self) 452 | 453 | signal.signal(sig, sig_handler) 454 | 455 | def _reset_signal_handlers(self): 456 | # type: () -> None 457 | for sig, sig_handler in self._dfl_sigmap.items(): 458 | signal.signal(sig, sig_handler) 459 | 460 | @staticmethod 461 | def _hide_cursor(target=None): 462 | # type: (Optional[IO]) -> None 463 | if not target: 464 | target = sys.stdout 465 | hide_cursor(stream=target) 466 | 467 | @staticmethod 468 | def _show_cursor(target=None): 469 | # type: (Optional[IO]) -> None 470 | if not target: 471 | target = sys.stdout 472 | show_cursor(stream=target) 473 | 474 | @staticmethod 475 | def _clear_err(): 476 | # type: () -> None 477 | sys.stderr.write(CLEAR_LINE) 478 | 479 | @staticmethod 480 | def _clear_line(): 481 | # type: () -> None 482 | sys.stdout.write(CLEAR_LINE) 483 | -------------------------------------------------------------------------------- /src/vistir/termcolors.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | import os 3 | import re 4 | 5 | import colorama 6 | 7 | from .misc import to_text as to_native_string 8 | 9 | DISABLE_COLORS = os.getenv("CI", False) or os.getenv( 10 | "ANSI_COLORS_DISABLED", os.getenv("VISTIR_DISABLE_COLORS", False) 11 | ) 12 | 13 | 14 | ATTRIBUTE_NAMES = ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"] 15 | ATTRIBUTES = dict(zip(ATTRIBUTE_NAMES, range(1, 9))) 16 | del ATTRIBUTES[""] 17 | 18 | colors = ["grey", "red", "green", "yellow", "blue", "magenta", "cyan", "white"] 19 | COLORS = dict(zip(colors, range(30, 38))) 20 | HIGHLIGHTS = dict(zip(["on_{0}".format(c) for c in colors], range(40, 48))) 21 | ANSI_REMOVAL_RE = re.compile(r"\033\[((?:\d|;)*)([a-zA-Z])") 22 | 23 | 24 | COLOR_MAP = { 25 | # name: type 26 | "blink": "attrs", 27 | "bold": "attrs", 28 | "concealed": "attrs", 29 | "dark": "attrs", 30 | "reverse": "attrs", 31 | "underline": "attrs", 32 | "blue": "color", 33 | "cyan": "color", 34 | "green": "color", 35 | "magenta": "color", 36 | "red": "color", 37 | "white": "color", 38 | "yellow": "color", 39 | "on_blue": "on_color", 40 | "on_cyan": "on_color", 41 | "on_green": "on_color", 42 | "on_grey": "on_color", 43 | "on_magenta": "on_color", 44 | "on_red": "on_color", 45 | "on_white": "on_color", 46 | "on_yellow": "on_color", 47 | } 48 | COLOR_ATTRS = COLOR_MAP.keys() 49 | 50 | 51 | RESET = colorama.Style.RESET_ALL 52 | 53 | 54 | def colored(text, color=None, on_color=None, attrs=None): 55 | """Colorize text using a reimplementation of the colorizer from 56 | https://github.com/pavdmyt/yaspin so that it works on windows. 57 | 58 | Available text colors: 59 | red, green, yellow, blue, magenta, cyan, white. 60 | 61 | Available text highlights: 62 | on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. 63 | 64 | Available attributes: 65 | bold, dark, underline, blink, reverse, concealed. 66 | 67 | Example: 68 | colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) 69 | colored('Hello, World!', 'green') 70 | """ 71 | return colorize(text, fg=color, bg=on_color, attrs=attrs) 72 | 73 | 74 | def colorize(text, fg=None, bg=None, attrs=None): 75 | if os.getenv("ANSI_COLORS_DISABLED") is None: 76 | style = "NORMAL" 77 | if attrs is not None and not isinstance(attrs, list): 78 | _attrs = [] 79 | if isinstance(attrs, str): 80 | _attrs.append(attrs) 81 | else: 82 | _attrs = list(attrs) 83 | attrs = _attrs 84 | if attrs and "bold" in attrs: 85 | style = "BRIGHT" 86 | attrs.remove("bold") 87 | if fg is not None: 88 | fg = fg.upper() 89 | text = to_native_string("%s%s%s%s%s") % ( 90 | to_native_string(getattr(colorama.Fore, fg)), 91 | to_native_string(getattr(colorama.Style, style)), 92 | to_native_string(text), 93 | to_native_string(colorama.Fore.RESET), 94 | to_native_string(colorama.Style.NORMAL), 95 | ) 96 | 97 | if bg is not None: 98 | bg = bg.upper() 99 | text = to_native_string("%s%s%s%s") % ( 100 | to_native_string(getattr(colorama.Back, bg)), 101 | to_native_string(text), 102 | to_native_string(colorama.Back.RESET), 103 | to_native_string(colorama.Style.NORMAL), 104 | ) 105 | 106 | if attrs is not None: 107 | fmt_str = to_native_string("%s[%%dm%%s%s[9m") % (chr(27), chr(27)) 108 | for attr in attrs: 109 | text = fmt_str % (ATTRIBUTES[attr], text) 110 | 111 | text += RESET 112 | else: 113 | text = ANSI_REMOVAL_RE.sub("", text) 114 | return text 115 | 116 | 117 | def cprint(text, color=None, on_color=None, attrs=None, **kwargs): 118 | """Print colorize text. 119 | 120 | It accepts arguments of print function. 121 | """ 122 | 123 | print((colored(text, color, on_color, attrs)), **kwargs) 124 | -------------------------------------------------------------------------------- /tasks/CHANGELOG.rst.jinja2: -------------------------------------------------------------------------------- 1 | {% for section in sections %} 2 | {% set underline = "-" %} 3 | {% if section %} 4 | {{section}} 5 | {{ underline * section|length }}{% set underline = "~" %} 6 | 7 | {% endif %} 8 | {% if sections[section] %} 9 | {% for category, val in definitions.items() if category in sections[section] and category != 'trivial' %} 10 | 11 | {{ definitions[category]['name'] }} 12 | {{ underline * definitions[category]['name']|length }} 13 | 14 | {% if definitions[category]['showcontent'] %} 15 | {% for text, values in sections[section][category]|dictsort(by='value') %} 16 | - {{ text }}{% if category != 'process' %} 17 | {{ values|sort|join(',\n ') }} 18 | {% endif %} 19 | 20 | {% endfor %} 21 | {% else %} 22 | - {{ sections[section][category]['']|sort|join(', ') }} 23 | 24 | 25 | {% endif %} 26 | {% if sections[section][category]|length == 0 %} 27 | 28 | No significant changes. 29 | 30 | 31 | {% else %} 32 | {% endif %} 33 | {% endfor %} 34 | {% else %} 35 | 36 | No significant changes. 37 | 38 | 39 | {% endif %} 40 | {% endfor %} 41 | -------------------------------------------------------------------------------- /tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | import datetime 3 | import pathlib 4 | import re 5 | import shutil 6 | import subprocess 7 | import time 8 | 9 | import invoke 10 | import parver 11 | from towncrier._builder import find_fragments, render_fragments, split_fragments 12 | from towncrier._settings import load_config 13 | 14 | 15 | def _get_git_root(ctx): 16 | return pathlib.Path( 17 | ctx.run("git rev-parse --show-toplevel", hide=True).stdout.strip() 18 | ) 19 | 20 | 21 | def _get_branch(ctx): 22 | return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip() 23 | 24 | 25 | ROOT = pathlib.Path(__file__).resolve().parent.parent 26 | 27 | PACKAGE_NAME = "vistir" 28 | 29 | INIT_PY = ROOT.joinpath("src", PACKAGE_NAME, "__init__.py") 30 | 31 | 32 | @invoke.task() 33 | def clean(ctx): 34 | """Clean previously built package artifacts. 35 | """ 36 | dist = ROOT.joinpath("dist") 37 | build = ROOT.joinpath("build") 38 | print("[clean] Removing dist and build dirs") 39 | if dist.exists(): 40 | shutil.rmtree(dist.as_posix()) 41 | if build.exists(): 42 | shutil.rmtree(build.as_posix()) 43 | 44 | 45 | def _read_version(): 46 | out = subprocess.check_output(["git", "tag"], encoding="ascii") 47 | try: 48 | version = max( 49 | parver.Version.parse(v).normalize() 50 | for v in (line.strip() for line in out.split("\n")) 51 | if v 52 | ) 53 | except ValueError: 54 | version = parver.Version.parse("0.0.0") 55 | return version 56 | 57 | 58 | def _read_text_version(): 59 | lines = INIT_PY.read_text().splitlines() 60 | match = next(iter(line for line in lines if line.startswith("__version__")), None) 61 | if match is not None: 62 | _, _, version_text = match.partition("=") 63 | version_text = version_text.strip().strip('"').strip("'") 64 | version = parver.Version.parse(version_text).normalize() 65 | return version 66 | else: 67 | return _read_version() 68 | 69 | 70 | def _write_version(v): 71 | lines = [] 72 | with INIT_PY.open() as f: 73 | for line in f: 74 | if line.startswith("__version__ = "): 75 | line = f"__version__ = {repr(str(v))}\n".replace("'", '"') 76 | lines.append(line) 77 | with INIT_PY.open("w", newline="\n") as f: 78 | f.write("".join(lines)) 79 | 80 | 81 | def _render_log(): 82 | """Totally tap into Towncrier internals to get an in-memory result. 83 | """ 84 | config = load_config(ROOT) 85 | definitions = config["types"] 86 | fragments, fragment_filenames = find_fragments( 87 | pathlib.Path(config["directory"]).absolute(), 88 | config["sections"], 89 | None, 90 | definitions, 91 | ) 92 | rendered = render_fragments( 93 | pathlib.Path(config["template"]).read_text(encoding="utf-8"), 94 | config["issue_format"], 95 | split_fragments(fragments, definitions), 96 | definitions, 97 | config["underlines"][1:], 98 | False, # Don't add newlines to wrapped text. 99 | { 100 | "name": "vistir", 101 | "version": "0.5.6", 102 | "date": "2022-7-27", 103 | }, # towncrier==19.9.0 104 | ) 105 | return rendered 106 | 107 | 108 | REL_TYPES = ("major", "minor", "patch") 109 | 110 | 111 | def _bump_release(version, type_, log=False): 112 | if type_ not in REL_TYPES: 113 | raise ValueError(f"{type_} not in {REL_TYPES}") 114 | index = REL_TYPES.index(type_) 115 | current_version = version.base_version() 116 | if version.is_prerelease and type_ == "patch": 117 | next_version = current_version 118 | else: 119 | next_version = current_version.bump_release(index=index) 120 | if log: 121 | print(f"[bump] {version} -> {next_version}") 122 | print(f"{next_version}") 123 | return next_version 124 | 125 | 126 | def _prebump(version, prebump, log=False): 127 | next_version = version.bump_release(index=prebump).bump_dev() 128 | if log: 129 | print(f"[bump] {version} -> {next_version}") 130 | print(f"{next_version}") 131 | return next_version 132 | 133 | 134 | PREBUMP = "patch" 135 | 136 | 137 | @invoke.task(pre=[clean]) 138 | def build(ctx): 139 | ctx.run("python setup.py sdist bdist_wheel") 140 | 141 | 142 | @invoke.task() 143 | def get_next_version(ctx, type_="patch", log=False): 144 | version = _read_version() 145 | if type_ in ("dev", "pre"): 146 | idx = REL_TYPES.index("patch") 147 | new_version = _prebump(version, idx, log=log) 148 | else: 149 | new_version = _bump_release(version, type_, log=log) 150 | return new_version 151 | 152 | 153 | @invoke.task() 154 | def bump_version(ctx, type_="patch", log=False, dry_run=False): 155 | new_version = get_next_version(ctx, type_, log=log) 156 | if not dry_run: 157 | _write_version(new_version) 158 | return new_version 159 | 160 | 161 | @invoke.task() 162 | def generate_news(ctx, yes=False, dry_run=False): 163 | command = "towncrier" 164 | if dry_run: 165 | command = f"{command} --draft" 166 | elif yes: 167 | command = f"{command} --yes" 168 | ctx.run(command) 169 | 170 | 171 | @invoke.task() 172 | def get_changelog(ctx): 173 | changelog = _render_log() 174 | print(changelog) 175 | return changelog 176 | 177 | 178 | @invoke.task(optional=["version", "type_"]) 179 | def tag_release(ctx, version=None, type_="patch", yes=False, dry_run=False): 180 | if version is None: 181 | version = bump_version(ctx, type_, log=not dry_run, dry_run=dry_run) 182 | else: 183 | _write_version(version) 184 | tag_content = get_changelog(ctx) 185 | generate_news(ctx, yes=yes, dry_run=dry_run) 186 | git_commit_cmd = f'git commit -am "Release {version}"' 187 | tag_content = tag_content.replace('"', '\\"') 188 | git_tag_cmd = f'git tag -a {version} -m "Version {version}\n\n{tag_content}"' 189 | if dry_run: 190 | print("Would run commands:") 191 | print(f" {git_commit_cmd}") 192 | print(f" {git_tag_cmd}") 193 | else: 194 | ctx.run(git_commit_cmd) 195 | ctx.run(git_tag_cmd) 196 | 197 | 198 | @invoke.task(optional=["version", "type_"]) 199 | def release(ctx, version=None, type_="patch", yes=False, dry_run=False): 200 | if version is None: 201 | version = bump_version(ctx, type_, log=not dry_run, dry_run=dry_run) 202 | else: 203 | _write_version(version) 204 | tag_content = get_changelog(ctx) 205 | current_branch = _get_branch(ctx) 206 | generate_news(ctx, yes=yes, dry_run=dry_run) 207 | git_commit_cmd = f'git commit -am "Release {version}"' 208 | git_tag_cmd = f'git tag -a {version} -m "Version {version}\n\n{tag_content}"' 209 | git_push_cmd = f"git push origin {current_branch}" 210 | git_push_tags_cmd = "git push --tags" 211 | if dry_run: 212 | print("Would run commands:") 213 | print(f" {git_commit_cmd}") 214 | print(f" {git_tag_cmd}") 215 | print(f" {git_push_cmd}") 216 | print(f" {git_push_tags_cmd}") 217 | else: 218 | ctx.run(git_commit_cmd) 219 | ctx.run(git_tag_cmd) 220 | ctx.run(git_push_cmd) 221 | print("Waiting 5 seconds before pushing tags...") 222 | time.sleep(5) 223 | ctx.run(git_push_tags_cmd) 224 | 225 | 226 | @invoke.task(pre=[clean]) 227 | def full_release(ctx, type_, repo, prebump=PREBUMP, yes=False): 228 | """Make a new release. 229 | """ 230 | if prebump not in REL_TYPES: 231 | raise ValueError(f"{type_} not in {REL_TYPES}") 232 | prebump = REL_TYPES.index(prebump) 233 | 234 | version = bump_version(ctx, type_, log=True) 235 | 236 | # Needs to happen before Towncrier deletes fragment files. 237 | 238 | tag_release(version, yes=yes) 239 | 240 | ctx.run(f"python setup.py sdist bdist_wheel") 241 | 242 | dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' 243 | artifacts = list(ROOT.joinpath("dist").glob(dist_pattern)) 244 | filename_display = "\n".join(f" {a}" for a in artifacts) 245 | print(f"[release] Will upload:\n{filename_display}") 246 | if not yes: 247 | try: 248 | input("[release] Release ready. ENTER to upload, CTRL-C to abort: ") 249 | except KeyboardInterrupt: 250 | print("\nAborted!") 251 | return 252 | 253 | arg_display = " ".join(f'"{n}"' for n in artifacts) 254 | ctx.run(f'twine upload --repository="{repo}" {arg_display}') 255 | 256 | version = _prebump(version, prebump) 257 | _write_version(version) 258 | 259 | ctx.run(f'git commit -am "Prebump to {version}"') 260 | 261 | 262 | @invoke.task 263 | def build_docs(ctx): 264 | _current_version = _read_text_version() 265 | minor = [str(i) for i in _current_version.release[:2]] 266 | docs_folder = (_get_git_root(ctx) / "docs").as_posix() 267 | if not docs_folder.endswith("/"): 268 | docs_folder = "{0}/".format(docs_folder) 269 | args = ["--ext-autodoc", "--ext-viewcode", "-o", docs_folder] 270 | args.extend(["-A", "'Dan Ryan '"]) 271 | args.extend(["-R", str(_current_version)]) 272 | args.extend(["-V", ".".join(minor)]) 273 | args.extend(["-e", "-M", "-F", f"src/{PACKAGE_NAME}"]) 274 | print("Building docs...") 275 | ctx.run("sphinx-apidoc {0}".format(" ".join(args))) 276 | 277 | 278 | @invoke.task 279 | def clean_mdchangelog(ctx): 280 | root = _get_git_root(ctx) 281 | changelog = root / "CHANGELOG.md" 282 | content = changelog.read_text() 283 | content = re.sub( 284 | r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/sarugaku/[\w\-]+/issues/\d+\)", 285 | r"\1 \2", 286 | content, 287 | flags=re.MULTILINE, 288 | ) 289 | changelog.write_text(content) 290 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | import vistir 4 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | import pytest 3 | from vistir.contextmanagers import replaced_stream 4 | 5 | 6 | @pytest.fixture(scope="function") 7 | def capture_streams(): 8 | with replaced_stream("stdout") as stdout: 9 | with replaced_stream("stderr") as stderr: 10 | yield (stdout, stderr) 11 | -------------------------------------------------------------------------------- /tests/strategies.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import codecs 5 | import os 6 | from collections import namedtuple 7 | 8 | from hypothesis import strategies as st 9 | from urllib import parse as urllib_parse 10 | 11 | from vistir.misc import to_text 12 | 13 | parsed_url = namedtuple("ParsedUrl", "scheme netloc path params query fragment") 14 | parsed_url.__new__.__defaults__ = ("", "", "", "", "", "") 15 | relative_path = namedtuple("RelativePath", "leading_dots separator dest") 16 | relative_path.__new__.__defaults__ = ("", "", "") 17 | url_alphabet = "abcdefghijklmnopqrstuvwxyz1234567890-" 18 | uri_schemes = ("http", "https", "ssh", "file", "sftp", "ftp") 19 | vcs_schemes = ( 20 | "git", 21 | "git+http", 22 | "git+https", 23 | "git+ssh", 24 | "git+git", 25 | "git+file", 26 | "hg", 27 | "hg+http", 28 | "hg+https", 29 | "hg+ssh", 30 | "hg+static-http", 31 | "svn", 32 | "svn+ssh", 33 | "svn+http", 34 | "svn+https", 35 | "svn+svn", 36 | "bzr", 37 | "bzr+http", 38 | "bzr+https", 39 | "bzr+ssh", 40 | "bzr+sftp", 41 | "bzr+ftp", 42 | "bzr+lp", 43 | ) 44 | 45 | 46 | # from https://github.com/twisted/txacme/blob/master/src/txacme/test/strategies.py 47 | def dns_labels(): 48 | """ 49 | Strategy for generating limited charset DNS labels. 50 | """ 51 | # This is too limited, but whatever 52 | return st.text( 53 | "abcdefghijklmnopqrstuvwxyz0123456789-", min_size=1, max_size=25 54 | ).filter( 55 | lambda s: not any( 56 | [s.startswith("-"), s.endswith("-"), s.isdigit(), s[2:4] == "--"] 57 | ) 58 | ) 59 | 60 | 61 | def dns_names(): 62 | """ 63 | Strategy for generating limited charset DNS names. 64 | """ 65 | return st.lists(dns_labels(), min_size=1, max_size=10).map(".".join) 66 | 67 | 68 | def urls(): 69 | """ 70 | Strategy for generating urls. 71 | """ 72 | return st.builds( 73 | parsed_url, 74 | scheme=st.sampled_from(uri_schemes), 75 | netloc=dns_names(), 76 | path=st.lists( 77 | st.text( 78 | max_size=64, 79 | alphabet=st.characters( 80 | blacklist_characters="/?#", blacklist_categories=("Cs",) 81 | ), 82 | ), 83 | min_size=1, 84 | max_size=10, 85 | ) 86 | .map(to_text) 87 | .map("".join), 88 | ) 89 | 90 | 91 | def legal_path_chars(): 92 | # Control characters 93 | blacklist = ["/"] 94 | if os.name == "nt": 95 | blacklist.extend(["<", ">", ":", '"', "\\", "|", "?", "*"]) 96 | return st.text( 97 | st.characters( 98 | blacklist_characters=blacklist, blacklist_categories=("C",), min_codepoint=32, 99 | ), 100 | min_size=1, 101 | max_size=10, 102 | ) 103 | 104 | 105 | def path_strategy(): 106 | return ( 107 | st.lists(legal_path_chars(), min_size=1, max_size=200) 108 | .map("".join) 109 | .filter(lambda s: not any(s.endswith(c) for c in [".", "..", " "])) 110 | .filter(lambda s: not s.startswith(" ")) 111 | .filter(lambda s: len(s) < 255) 112 | ) 113 | 114 | 115 | @st.composite 116 | def paths(draw, path=path_strategy()): 117 | path_part = draw(path) 118 | return path_part 119 | 120 | 121 | def relative_paths(): 122 | relative_leaders = (".", "..") 123 | separators = [ 124 | to_text(sep) for sep in (os.sep, os.path.sep, os.path.altsep) if sep is not None 125 | ] 126 | return st.builds( 127 | relative_path, 128 | leading_dots=st.sampled_from(relative_leaders), 129 | separator=st.sampled_from(separators), 130 | dest=legal_path_chars(), 131 | ) 132 | 133 | 134 | def unparsed_urls(): 135 | return st.builds(urllib_parse.urlunparse, urls()) 136 | -------------------------------------------------------------------------------- /tests/test_contextmanagers.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, print_function, unicode_literals 3 | 4 | import contextlib 5 | import importlib 6 | import io 7 | import os 8 | import shutil 9 | import sys 10 | import warnings 11 | 12 | import pytest 13 | 14 | from vistir import contextmanagers 15 | 16 | from .utils import read_file 17 | 18 | from unittest.mock import patch 19 | 20 | 21 | GUTENBERG_FILE = os.path.join( 22 | os.path.dirname(os.path.abspath(__file__)), "fixtures/gutenberg_document.txt" 23 | ) 24 | 25 | 26 | def is_fp_closed(fp): 27 | try: 28 | return fp.isclosed() 29 | except AttributeError: 30 | pass 31 | try: 32 | return fp.closed 33 | except AttributeError: 34 | pass 35 | try: 36 | return fp.fp is None 37 | except AttributeError: 38 | pass 39 | raise ValueError("could not find fp on object") 40 | 41 | 42 | class MockUrllib3Response(object): 43 | def __init__(self, path): 44 | self._fp = io.open(path, "rb") 45 | self._fp_bytes_read = 0 46 | self.auto_close = True 47 | self.length_remaining = 799738 48 | 49 | def __enter__(self): 50 | return self 51 | 52 | def __exit__(self, *args, **kwargs): 53 | self.close() 54 | 55 | def stream(self, amt=2 ** 16, decode_content=None): 56 | while not is_fp_closed(self._fp): 57 | data = self.read(amt=amt, decode_content=decode_content) 58 | if data: 59 | yield data 60 | 61 | def close(self): 62 | if not self.closed: 63 | self._fp.close() 64 | 65 | @property 66 | def closed(self): 67 | if not self.auto_close: 68 | return io.IOBase.closed.__get__(self) 69 | elif self._fp is None: 70 | return True 71 | elif hasattr(self._fp, "isclosed"): 72 | return self._fp.isclosed() 73 | elif hasattr(self._fp, "closed"): 74 | return self._fp.closed 75 | else: 76 | return True 77 | 78 | def fileno(self): 79 | if self._fp is None: 80 | raise IOError("HTTPResponse has no file to get a fileno from") 81 | elif hasattr(self._fp, "fileno"): 82 | return self._fp.fileno() 83 | else: 84 | raise IOError( 85 | "The file-like object this HTTPResponse is wrapped " 86 | "around has no file descriptor" 87 | ) 88 | 89 | def readinto(self, b): 90 | # This method is required for `io` module compatibility. 91 | temp = self.read(len(b)) 92 | if len(temp) == 0: 93 | return 0 94 | else: 95 | b[: len(temp)] = temp 96 | return len(temp) 97 | 98 | @property 99 | def data(self): 100 | if self._body: 101 | return self._body 102 | if self._fp: 103 | return self.read(cache_content=True) 104 | 105 | def read(self, amt=None, decode_content=None, cache_content=False): 106 | if self._fp is None: 107 | return 108 | fp_closed = getattr(self._fp, "closed", False) 109 | if amt is None: 110 | # cStringIO doesn't like amt=None 111 | data = self._fp.read() if not fp_closed else b"" 112 | flush_decoder = True 113 | else: 114 | cache_content = False 115 | data = self._fp.read(amt) if not fp_closed else b"" 116 | if amt != 0 and not data: # Platform-specific: Buggy versions of Python. 117 | self._fp.close() 118 | 119 | if data: 120 | self._fp_bytes_read += len(data) 121 | if self.length_remaining is not None: 122 | self.length_remaining -= len(data) 123 | 124 | if cache_content: 125 | self._body = data 126 | 127 | return data 128 | 129 | def isclosed(self): 130 | return is_fp_closed(self._fp) 131 | 132 | def tell(self): 133 | """ 134 | Obtain the number of bytes pulled over the wire so far. May differ from 135 | the amount of content returned by :meth:``HTTPResponse.read`` if bytes 136 | are encoded on the wire (e.g, compressed). 137 | """ 138 | return self._fp_bytes_read 139 | 140 | def __iter__(self): 141 | buffer = [] 142 | for chunk in self.stream(decode_content=True): 143 | if b"\n" in chunk: 144 | chunk = chunk.split(b"\n") 145 | yield b"".join(buffer) + chunk[0] + b"\n" 146 | for x in chunk[1:-1]: 147 | yield x + b"\n" 148 | if chunk[-1]: 149 | buffer = [chunk[-1]] 150 | else: 151 | buffer = [] 152 | else: 153 | buffer.append(chunk) 154 | if buffer: 155 | yield b"".join(buffer) 156 | 157 | 158 | def test_path(): 159 | old_path = sys.path 160 | new_path = sys.path[2:] 161 | with contextmanagers.temp_path(): 162 | sys.path = new_path 163 | assert sys.path == new_path 164 | assert sys.path == old_path 165 | 166 | 167 | def test_cd(tmpdir): 168 | second_dir = tmpdir.join("second_dir").mkdir() 169 | original_dir = os.path.abspath(os.curdir) 170 | assert original_dir != os.path.abspath(second_dir.strpath) 171 | with contextmanagers.cd(second_dir.strpath): 172 | assert os.path.abspath(os.curdir) == os.path.abspath(second_dir.strpath) 173 | assert os.path.abspath(os.curdir) == original_dir 174 | 175 | 176 | def test_environ(): 177 | os.environ["VISTIR_TEST_KEY"] = "test_value" 178 | assert os.environ.get("VISTIR_OTHER_KEY") is None 179 | with contextmanagers.temp_environ(): 180 | os.environ["VISTIR_TEST_KEY"] = "temporary test value" 181 | os.environ["VISTIR_OTHER_KEY"] = "some_other_key_value" 182 | assert os.environ["VISTIR_TEST_KEY"] == "temporary test value" 183 | assert os.environ["VISTIR_OTHER_KEY"] == "some_other_key_value" 184 | assert os.environ["VISTIR_TEST_KEY"] == "test_value" 185 | assert os.environ.get("VISTIR_OTHER_KEY") is None 186 | 187 | 188 | def test_atomic_open(tmpdir, monkeypatch): 189 | test_file = tmpdir.join("test_file.txt") 190 | replace_with_text = "new test text" 191 | test_file.write_text("some test text", encoding="utf-8") 192 | assert read_file(test_file.strpath) == "some test text" 193 | 194 | # Raise an error to make sure we don't write text on errors 195 | def raise_exception_while_writing(filename, new_text): 196 | with contextmanagers.atomic_open_for_write(filename) as fh: 197 | fh.write(new_text) 198 | raise RuntimeError("This should not overwrite the file") 199 | 200 | def raise_oserror_on_chmod(path, mode, dir_fd=None, follow_symlinks=True): 201 | raise OSError("No permission!") 202 | 203 | try: 204 | raise_exception_while_writing(test_file.strpath, replace_with_text) 205 | except RuntimeError: 206 | pass 207 | # verify that the file is not overwritten 208 | assert read_file(test_file.strpath) == "some test text" 209 | with contextmanagers.atomic_open_for_write(test_file.strpath) as fh: 210 | fh.write(replace_with_text) 211 | # make sure that we now have the new text in the file 212 | assert read_file(test_file.strpath) == replace_with_text 213 | another_test_file = tmpdir.join("test_file_for_exceptions.txt") 214 | another_test_file.write_text("original test text", encoding="utf-8") 215 | more_text = "this is more test text" 216 | with monkeypatch.context() as m: 217 | m.setattr(os, "chmod", raise_oserror_on_chmod) 218 | with contextmanagers.atomic_open_for_write( 219 | another_test_file.strpath 220 | ) as fh: 221 | fh.write(more_text) 222 | assert read_file(another_test_file.strpath) == more_text 223 | 224 | 225 | class MockLink(object): 226 | def __init__(self, url): 227 | self.url = url 228 | 229 | @property 230 | def url_without_fragment(self): 231 | return self.url 232 | 233 | 234 | @pytest.mark.parametrize( 235 | "stream, use_requests, use_link", 236 | [ 237 | (True, True, True), 238 | (True, True, False), 239 | (True, False, True), 240 | (True, False, False), 241 | (False, True, True), 242 | (False, True, False), 243 | (False, False, True), 244 | (False, False, False), 245 | ], 246 | ) 247 | def test_open_file_without_requests(monkeypatch, tmpdir, stream, use_requests, use_link): 248 | 249 | module_prefix = "builtins" 250 | bi = importlib.import_module(module_prefix) 251 | import_func = bi.__import__ 252 | del bi 253 | 254 | def _import(name, globals=None, locals=None, fromlist=(), level=0): 255 | if not use_requests and name.startswith("requests"): 256 | raise ImportError(name) 257 | return import_func(name, globals, locals, fromlist, level) 258 | 259 | warnings.filterwarnings( 260 | "ignore", category=ResourceWarning, message="unclosed.*" 261 | ) 262 | if stream: 263 | target_file = ( 264 | "https://www2.census.gov/geo/tiger/GENZ2017/shp/cb_2017_02_tract_500k.zip" 265 | ) 266 | else: 267 | target_file = "https://www.fakegutenberg.org/files/1342/1342-0.txt" 268 | if use_link: 269 | target_file = MockLink(target_file) 270 | filecontents = io.BytesIO(b"") 271 | module_name = "{0}.__import__".format(module_prefix) 272 | 273 | @contextlib.contextmanager 274 | def patch_context(): 275 | if not stream and use_requests: 276 | import requests 277 | 278 | with patch( 279 | "requests.Session.get", return_value=MockUrllib3Response(GUTENBERG_FILE) 280 | ): 281 | yield 282 | elif stream and not use_requests: 283 | 284 | import http 285 | 286 | with patch( 287 | "http.client.HTTPSConnection.request", 288 | return_value=MockUrllib3Response(GUTENBERG_FILE), 289 | ), patch( 290 | "urllib.request.urlopen", 291 | return_value=MockUrllib3Response(GUTENBERG_FILE), 292 | ): 293 | yield 294 | 295 | else: 296 | yield 297 | 298 | with monkeypatch.context() as m, patch_context(): 299 | if not use_requests: 300 | m.delitem(sys.modules, "requests", raising=False) 301 | m.delitem(sys.modules, "requests.sessions", raising=False) 302 | import urllib.request 303 | 304 | def do_urlopen(*args, **kwargs): 305 | return MockUrllib3Response(GUTENBERG_FILE) 306 | 307 | m.setattr(urllib.request, "urlopen", do_urlopen) 308 | 309 | with patch(module_name, _import): 310 | with contextmanagers.open_file(target_file, stream=stream) as fp: 311 | if stream: 312 | shutil.copyfileobj(fp, filecontents) 313 | else: 314 | filecontents.write(fp.read()) 315 | local_file = tmpdir.join("local_copy.txt") 316 | with io.open(local_file.strpath, "w", encoding="utf-8") as fp: 317 | fp.write(filecontents.read().decode()) 318 | 319 | local_contents = b"" 320 | with contextmanagers.open_file(local_file.strpath) as fp: 321 | for chunk in iter(lambda: fp.read(16384), b""): 322 | local_contents += chunk 323 | assert local_contents == filecontents.read() 324 | 325 | 326 | def test_replace_stream(capsys): 327 | with contextmanagers.replaced_stream("stdout") as stdout: 328 | sys.stdout.write("hello") 329 | assert stdout.getvalue() == "hello" 330 | out, err = capsys.readouterr() 331 | assert out.strip() != "hello" 332 | 333 | 334 | def test_replace_streams(capsys): 335 | with contextmanagers.replaced_streams() as streams: 336 | stdout, stderr = streams 337 | sys.stdout.write("hello") 338 | sys.stderr.write("this is an error") 339 | assert stdout.getvalue() == "hello" 340 | assert stderr.getvalue() == "this is an error" 341 | out, err = capsys.readouterr() 342 | assert out.strip() != "hello" 343 | assert err.strip() != "this is an error" 344 | -------------------------------------------------------------------------------- /tests/test_misc.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | 3 | import io 4 | import itertools 5 | import locale 6 | import os 7 | import sys 8 | 9 | import pytest 10 | from hypothesis import assume, given, strategies as st 11 | 12 | import vistir 13 | 14 | from .strategies import legal_path_chars 15 | 16 | 17 | def test_get_logger(): 18 | from logging import Logger 19 | 20 | logger = vistir.misc._get_logger(name="vistir_logger", level="DEBUG") 21 | assert isinstance(logger, Logger) 22 | 23 | 24 | def test_shell_escape(): 25 | printfoo = "python -c \"print('foo')\"" 26 | assert vistir.misc.shell_escape(printfoo) == "python -c print('foo')" 27 | appendscript = "cmd arg1" 28 | assert vistir.misc.shell_escape(appendscript) == "cmd arg1" 29 | multicommand = u'bash -c "cd docs && make html"' 30 | assert vistir.misc.shell_escape(multicommand) == 'bash -c "cd docs && make html"' 31 | escaped_python = '"{}" -c \'print("hello")\''.format(sys.executable) 32 | if os.name == "nt" and " " in sys.executable: 33 | expected = '"{}" -c print("hello")'.format(sys.executable) 34 | else: 35 | expected = '{} -c print("hello")'.format(sys.executable) 36 | assert vistir.misc.shell_escape(escaped_python) == expected 37 | 38 | 39 | def test_get_stream_results(): 40 | class MockCmd(object): 41 | def __init__(self, stdout, stderr): 42 | self.stdout = stdout 43 | self.stderr = stderr 44 | 45 | def poll(self): 46 | return 0 47 | 48 | def wait(self): 49 | return 0 50 | 51 | stdout_buffer = io.StringIO() 52 | stderr_buffer = io.StringIO() 53 | test_line = ( 54 | u"this is a test line that goes on for many characters and will eventually be " 55 | "truncated because it is far too long to display on a normal terminal so we will" 56 | " use an ellipsis to break it\n" 57 | ) 58 | stdout_buffer.write(test_line) 59 | stdout_buffer.seek(0) 60 | cmd_instance = MockCmd(stdout=stdout_buffer, stderr=stderr_buffer) 61 | instance = vistir.misc.attach_stream_reader( 62 | cmd_instance, False, 50, spinner=None, stdout_allowed=False 63 | ) 64 | assert instance.text_stdout_lines == [test_line.strip()], "\n".join( 65 | ["{}: {}".format(k, v) for k, v in instance.__dict__.items()] 66 | ) 67 | 68 | 69 | def test_run(): 70 | out, err = vistir.misc.run( 71 | [r"{}".format(sys.executable), "-c", "print('hello')"], nospin=True 72 | ) 73 | assert out == "hello" 74 | out, err = vistir.misc.run( 75 | [sys.executable, "-c", "import ajwfoiejaoiwj"], nospin=True 76 | ) 77 | assert any( 78 | error_text in err for error_text in ["ImportError", "ModuleNotFoundError"] 79 | ), "{} => {}".format(out, err) 80 | 81 | 82 | def test_run_return_subprocess(): 83 | c = vistir.misc.run( 84 | [r"{}".format(sys.executable), "-c", "print('test')"], 85 | return_object=True, 86 | nospin=True, 87 | ) 88 | assert c.returncode == 0 89 | assert c.out.strip() == "test" 90 | 91 | 92 | @pytest.mark.flaky(reruns=5) 93 | def test_run_with_long_output(): 94 | long_str = "this is a very long string which exceeds the maximum length per the settings we are passing in to vistir" 95 | print_cmd = "import time; print('{}'); time.sleep(2)".format(long_str) 96 | run_args = [r"{}".format(sys.executable), "-c", print_cmd] 97 | c = vistir.misc.run( 98 | run_args, block=False, display_limit=100, nospin=True, return_object=True 99 | ) 100 | c.wait() 101 | assert c.out == long_str 102 | c = vistir.misc.run( 103 | run_args, 104 | block=False, 105 | display_limit=100, 106 | nospin=True, 107 | verbose=True, 108 | return_object=True, 109 | ) 110 | c.wait() 111 | assert c.out == long_str 112 | 113 | c = vistir.misc.run( 114 | run_args, 115 | block=False, 116 | write_to_stdout=False, 117 | nospin=True, 118 | verbose=True, 119 | return_object=True, 120 | ) 121 | c.wait() 122 | assert c.out == long_str 123 | 124 | 125 | @pytest.mark.flaky(reruns=5) 126 | def test_nonblocking_run(): 127 | c = vistir.misc.run( 128 | [r"{}".format(sys.executable), "--help"], 129 | block=False, 130 | return_object=True, 131 | nospin=True, 132 | ) 133 | assert c.returncode == 0 134 | c.wait() 135 | assert "PYTHONDONTWRITEBYTECODE" in c.out, c.out 136 | out, _ = vistir.misc.run( 137 | [r"{}".format(sys.executable), "--help"], block=False, nospin=True 138 | ) 139 | assert "PYTHONDONTWRITEBYTECODE" in out, out 140 | 141 | 142 | def test_load_path(): 143 | loaded_path = vistir.misc.load_path(sys.executable) 144 | assert any(sys.exec_prefix in loaded_sys_path for loaded_sys_path in loaded_path) 145 | 146 | 147 | def test_partialclass(): 148 | text_io_wrapper = vistir.misc.partialclass(io.TextIOWrapper) 149 | instantiated_wrapper = text_io_wrapper(io.BytesIO(b"hello")) 150 | assert instantiated_wrapper.read() == "hello" 151 | 152 | 153 | DIVIDE_ITERABLE = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 154 | 155 | 156 | def test_stream_wrapper(capsys): 157 | new_stream = vistir.misc.get_text_stream("stdout") 158 | sys.stdout = new_stream 159 | print("this is a new method\u0141asdf", file=sys.stdout) 160 | out, err = capsys.readouterr() 161 | assert out.strip() == "this is a new method\u0141asdf" 162 | -------------------------------------------------------------------------------- /tests/test_path.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import shutil 4 | import stat 5 | 6 | import pytest 7 | from hypothesis import HealthCheck, assume, example, given, settings 8 | from hypothesis_fspaths import _PathLike, fspaths 9 | from urllib import parse as urllib_parse 10 | 11 | import vistir 12 | 13 | from .strategies import legal_path_chars, paths, relative_paths, url_alphabet, urls 14 | from .utils import EXEC, NON_WRITE_OR_EXEC, NON_WRITEABLE, WRITEABLE, get_mode 15 | 16 | 17 | def get_ord(char): 18 | if isinstance(char, int): 19 | return char 20 | return ord(char) 21 | 22 | 23 | def test_abspathu(tmpdir): 24 | tmpdir.mkdir("new_dir") 25 | new_dir = tmpdir.join("new_dir") 26 | with vistir.contextmanagers.cd(tmpdir.strpath): 27 | assert vistir.path.abspathu(new_dir.purebasename) == vistir.misc.to_text( 28 | new_dir.strpath 29 | ) 30 | 31 | 32 | @pytest.mark.parametrize( 33 | "path, root, result", 34 | [ 35 | ("~/some/path/child", "~/some/path", True), 36 | ("~/some", "~/some/path", False), 37 | ("~/some/path/child", "~", True), 38 | ], 39 | ) 40 | def test_is_in_path(path, root, result): 41 | assert vistir.path.is_in_path(path, root) == result 42 | 43 | 44 | def test_safe_expandvars(): 45 | with vistir.contextmanagers.temp_environ(): 46 | os.environ["TEST_VAR"] = "some_password" 47 | expected = "https://myuser:some_password@myfakewebsite.com" 48 | sanitized = "https://myuser:${TEST_VAR}@myfakewebsite.com" 49 | assert vistir.path.safe_expandvars(sanitized) == expected 50 | 51 | 52 | @pytest.mark.xfail(raises=OSError) 53 | def test_mkdir_p_fails_when_path_exists(tmpdir): 54 | myfile = tmpdir.join("myfile") 55 | myfile.write_text(u"some text", encoding="utf-8") 56 | vistir.path.mkdir_p(myfile.strpath) 57 | 58 | 59 | def test_create_tracked_tempdir(tmpdir): 60 | subdir = tmpdir.join("subdir") 61 | subdir.mkdir() 62 | temp_dir = vistir.path.create_tracked_tempdir(prefix="test_dir", dir=subdir.strpath) 63 | assert os.path.basename(temp_dir).startswith("test_dir") 64 | with io.open(os.path.join(temp_dir, "test_file.txt"), "w") as fh: 65 | fh.write(u"this is a test") 66 | assert len(os.listdir(temp_dir)) > 0 67 | 68 | 69 | def test_create_tracked_tempfile(tmpdir): 70 | tmpfile = vistir.path.create_tracked_tempfile( 71 | prefix="test_file", dir=tmpdir.strpath, delete=False 72 | ) 73 | tmpfile.write(b"this is some text") 74 | tmpfile.close() 75 | assert os.path.basename(tmpfile.name).startswith("test_file") 76 | assert os.path.exists(tmpfile.name) 77 | with io.open(tmpfile.name, "r") as fh: 78 | contents = fh.read().strip() 79 | assert contents == "this is some text" 80 | 81 | 82 | def test_rmtree(tmpdir): 83 | """This will also test `handle_remove_readonly` and `set_write_bit`.""" 84 | new_dir = tmpdir.join("test_dir").mkdir() 85 | new_file = new_dir.join("test_file.py") 86 | new_file.write_text(u"some test text", encoding="utf-8") 87 | os.chmod(new_file.strpath, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) 88 | assert new_dir.exists() 89 | vistir.path.rmtree(new_dir.strpath) 90 | assert not new_dir.exists() 91 | 92 | 93 | def test_is_readonly_path(tmpdir): 94 | new_dir = tmpdir.join("some_dir").mkdir() 95 | new_file = new_dir.join("some_file.txt") 96 | new_file.write_text(u"this is some text", encoding="utf-8") 97 | assert not vistir.path.is_readonly_path(new_dir.strpath) 98 | assert not vistir.path.is_readonly_path(new_file.strpath) 99 | os.chmod(new_file.strpath, get_mode(new_file.strpath) & NON_WRITEABLE) 100 | assert vistir.path.is_readonly_path(new_file.strpath) 101 | assert vistir.path.is_readonly_path("fake_path") is False 102 | WRITE_EXEC = WRITEABLE | EXEC 103 | os.chmod(new_dir.strpath, WRITE_EXEC) 104 | os.chmod(new_file.strpath, WRITEABLE) 105 | shutil.rmtree(new_dir.strpath) 106 | 107 | 108 | @given(relative_paths()) 109 | def test_get_converted_relative_path(path): 110 | assume(not vistir.path.abspathu("".join(path)) == vistir.path.abspathu(os.curdir)) 111 | path = "".join(path) 112 | relpath = vistir.path.get_converted_relative_path(path) 113 | assert relpath.startswith(".") 114 | assert vistir.path.abspathu(relpath) == os.path.abspath(vistir.misc.to_text(path)) 115 | 116 | 117 | @given(urls()) 118 | def test_is_valid_url(url): 119 | unparsed_url = urllib_parse.urlunparse(url) 120 | assume(not unparsed_url.startswith("file://")) 121 | assert vistir.path.is_valid_url(unparsed_url) 122 | 123 | 124 | def test_none_as_valid_url_returns_none(): 125 | assert vistir.path.is_valid_url(None) is None 126 | 127 | 128 | def test_is_file_url_returns_none_when_passed_none(): 129 | assert vistir.path.is_file_url(None) is False 130 | 131 | 132 | @pytest.mark.xfail(raises=ValueError) 133 | def test_is_file_url_raises_valueerror_when_no_url_attribute_found(): 134 | class FakeLink(object): 135 | def __init__(self, url_path): 136 | self.url_path = url_path 137 | 138 | assert not vistir.path.is_file_url(FakeLink(vistir.path.path_to_url("some/link"))) 139 | 140 | 141 | @given(fspaths()) 142 | @example("0\ud800") 143 | @settings(suppress_health_check=(HealthCheck.filter_too_much,)) 144 | def test_path_to_url(filepath): 145 | class FakeLink(object): 146 | def __init__(self, url=None): 147 | self.url = url 148 | 149 | filename = str(filepath) 150 | if filepath and filename: 151 | assume(any(letter in filename for letter in url_alphabet)) 152 | file_url = vistir.path.path_to_url(filename) 153 | if filename: 154 | assume(file_url != filename) 155 | assert file_url.startswith("file:") 156 | assert vistir.path.is_file_url(file_url) is True 157 | assert vistir.path.is_file_url(FakeLink(url=file_url)) is True 158 | else: 159 | assert file_url == filename 160 | assert not file_url 161 | 162 | 163 | @given(fspaths()) 164 | @settings(suppress_health_check=(HealthCheck.filter_too_much,)) 165 | def test_normalize_drive(filepath): 166 | filename = vistir.misc.to_text(filepath) 167 | if filepath and filename: 168 | assume(any(letter in filename for letter in url_alphabet)) 169 | if os.name == "nt": 170 | upper_drive = "C:" 171 | lower_drive = upper_drive.lower() 172 | lower_path = os.path.join(lower_drive, filename) 173 | upper_path = os.path.join(upper_drive, filename) 174 | assert vistir.path.normalize_drive(lower_path) == upper_path 175 | assert vistir.path.normalize_drive(upper_path) == upper_path 176 | assert vistir.path.normalize_drive(filename) == filename 177 | 178 | 179 | def test_walk_up(tmpdir): 180 | tmpdir.join("test.txt").write_text(u"some random text", encoding="utf-8") 181 | one_down = tmpdir.join("one_down").mkdir() 182 | one_down.join("test.txt").write_text(u"some random text", encoding="utf-8") 183 | one_down.join("test1.txt").write_text(u"some random text 2", encoding="utf-8") 184 | one_down.join("test2.txt").write_text(u"some random text 3", encoding="utf-8") 185 | two_down = one_down.join("two_down").mkdir() 186 | two_down.join("test2.txt").write_text(u"some random text", encoding="utf-8") 187 | two_down.join("test2_1.txt").write_text(u"some random text 2", encoding="utf-8") 188 | two_down.join("test2_2.txt").write_text(u"some random text 3", encoding="utf-8") 189 | expected = ( 190 | ( 191 | os.path.abspath(two_down.strpath), 192 | [], 193 | sorted(["test2.txt", "test2_1.txt", "test2_2.txt"]), 194 | ), 195 | ( 196 | os.path.abspath(one_down.strpath), 197 | sorted([two_down.purebasename]), 198 | sorted(["test.txt", "test1.txt", "test2.txt"]), 199 | ), 200 | ( 201 | os.path.abspath(tmpdir.strpath), 202 | sorted([one_down.purebasename]), 203 | sorted(["test.txt"]), 204 | ), 205 | ) 206 | walk_up = vistir.path.walk_up(two_down.strpath) 207 | for i in range(3): 208 | results = next(walk_up) 209 | results = (results[0], sorted(results[1]), sorted(results[2])) 210 | assert results == expected[i] 211 | 212 | 213 | def test_handle_remove_readonly(tmpdir): 214 | test_file = tmpdir.join("test_file.txt") 215 | test_file.write_text(u"a bunch of text", encoding="utf-8") 216 | os.chmod(test_file.strpath, NON_WRITE_OR_EXEC) 217 | fake_oserror = OSError(13, "Permission denied") 218 | fake_oserror.filename = test_file.strpath 219 | vistir.path.handle_remove_readonly( 220 | os.unlink, 221 | test_file.strpath, 222 | (fake_oserror.__class__, fake_oserror, "Fake traceback"), 223 | ) 224 | assert not os.path.exists(test_file.strpath) 225 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding=utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import os 5 | import stat 6 | 7 | READ_ONLY = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH 8 | NON_WRITEABLE = ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH 9 | NON_WRITE_OR_EXEC = NON_WRITEABLE & ~stat.S_IXUSR & ~stat.S_IXGRP & ~stat.S_IXOTH 10 | WRITEABLE = stat.S_IWUSR | stat.S_IWGRP | stat.S_IRUSR 11 | EXEC = stat.S_IXUSR 12 | 13 | 14 | def get_mode(fn): 15 | return stat.S_IMODE(os.lstat(fn).st_mode) 16 | 17 | 18 | def read_file(fn): 19 | contents = "" 20 | with open(fn, "r") as fh: 21 | contents = fh.read().strip() 22 | return contents 23 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | docs, packaging, py37, py38, py39, py310, py311, coverage-report 4 | 5 | [testenv] 6 | passenv = CI GIT_SSL_CAINFO 7 | setenv = 8 | LC_ALL = en_US.UTF-8 9 | deps = 10 | -e .[tests,requests] 11 | coverage 12 | commands = coverage run --parallel-mode -m pytest -v -x --timeout 300 [] 13 | install_command = python -m pip install {opts} {packages} 14 | usedevelop = True 15 | 16 | [testenv:coverage-report] 17 | deps = coverage 18 | skip_install = true 19 | commands = 20 | coverage combine 21 | # coverage report 22 | coverage html 23 | 24 | [testenv:docs] 25 | deps = 26 | -r{toxinidir}/docs/requirements.txt 27 | -e .[requests,tests] 28 | commands = 29 | sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html 30 | sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man 31 | 32 | [testenv:packaging] 33 | deps = 34 | check-manifest 35 | twine 36 | readme_renderer[md] 37 | commands = 38 | check-manifest 39 | python setup.py sdist bdist_wheel 40 | twine check dist/* 41 | --------------------------------------------------------------------------------