├── tests ├── __init__.py ├── test_extra.py ├── test_mmap_pypy.py └── test_mmap.py ├── setup.cfg ├── .gitignore ├── pyproject.toml ├── .travis.yml ├── MANIFEST.in ├── tox.ini ├── .github └── workflows │ └── main.yml ├── src ├── constants.pxd ├── constants.h └── fmmap.pyx ├── benchmarks └── grep.py ├── setup.py ├── README.rst └── LICENSE.txt /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | # This includes the license file(s) in the wheel. 3 | # https://wheel.readthedocs.io/en/stable/user_guide.html#including-license-files-in-the-generated-wheel-file 4 | license_files = LICENSE.txt 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # general things to ignore 2 | build/ 3 | dist/ 4 | *.egg-info/ 5 | *.egg 6 | *.py[cod] 7 | __pycache__/ 8 | *.so 9 | env/ 10 | 11 | # due to using tox and pytest 12 | .tox 13 | .cache 14 | 15 | # editors 16 | .idea 17 | *~ 18 | *.swp 19 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # These are the assumed default build requirements from pip: 3 | # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support 4 | requires = ["setuptools>=40.8.0", "wheel", "Cython"] 5 | build-backend = "setuptools.build_meta" 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | - "3.7" 8 | - "3.8" 9 | - "3.9-dev" 10 | 11 | jobs: 12 | allow_failures: 13 | - python: "3.9-dev" 14 | 15 | install: pip install tox-travis 16 | 17 | script: tox 18 | 19 | notifications: 20 | email: false 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include pyproject.toml 2 | 3 | include src/constants.h 4 | include src/constants.pxd 5 | include src/fmmap.pyx 6 | include .travis.yml 7 | include tox.ini 8 | recursive-include benchmarks *.py 9 | recursive-include tests *.py 10 | 11 | # Include the README 12 | include *.rst 13 | 14 | # Include the license file 15 | include LICENSE.txt 16 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{34,35,36,37,38,39} 3 | 4 | # At least tox 3.3.0 is needed for PEP 517/518 support. If tox is older, tox 5 | # will install a newer version in a separate environment. 6 | minversion = 3.3.0 7 | 8 | # Activate isolated build environment. tox will use a virtual environment 9 | # to build a source distribution from the source tree. For build tools and 10 | # arguments use the pyproject.toml file as specified in PEP-517 and PEP-518. 11 | isolated_build = true 12 | 13 | [testenv] 14 | deps = 15 | #check-manifest 16 | readme_renderer < 25.0 17 | pytest 18 | commands = 19 | #check-manifest 20 | python setup.py check -m -r -s 21 | py.test tests {posargs} 22 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: fmmap 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | continue-on-error: ${{ matrix.experimental }} 10 | strategy: 11 | matrix: 12 | os: [ubuntu-18.04, macos-latest] 13 | python: [3.5, 3.6, 3.7, 3.8] 14 | experimental: [false] 15 | include: 16 | - os: windows-latest 17 | python: 3.8 18 | experimental: true 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Setup Python 23 | uses: actions/setup-python@v1 24 | with: 25 | python-version: ${{ matrix.python }} 26 | - name: Install Tox and any other packages 27 | run: pip install tox 28 | - name: Run Tox 29 | run: tox -e py # Run tox using the version of Python in `PATH` 30 | -------------------------------------------------------------------------------- /src/constants.pxd: -------------------------------------------------------------------------------- 1 | # Some of the constants for madvise(2) are not guaranteed to be available. 2 | # Apart from kernel version, it might be configured out, and simply not defined 3 | # in the headers. While Cython lacks a #ifndef equivalent, we have to define a 4 | # few things in a header file so that the names are defined so that the 5 | # generated C file will compile. 6 | 7 | cdef extern from "constants.h": 8 | 9 | # feature availability flags: 10 | enum: MEMMEM 11 | enum: MEMRCHR 12 | 13 | 14 | # constants related to madvise(2): 15 | 16 | # common 17 | enum: FREE 18 | 19 | # Linux 20 | enum: HWPOISON 21 | enum: MERGEABLE 22 | enum: SOFT_OFFLINE 23 | enum: HUGEPAGE 24 | enum: DUMP 25 | enum: ONFORK 26 | 27 | # In the following cases, the MADV_* constants are not all defined in 28 | # Cython's mman.pxd, so we add them in addition to our feature flags. 29 | 30 | # FreeBSD 31 | enum: NOSYNC 32 | enum: MADV_NOSYNC 33 | 34 | enum: AUTOSYNC 35 | enum: MADV_AUTOSYNC 36 | 37 | enum: NOCORE 38 | enum: MADV_NOCORE 39 | 40 | enum: CORE 41 | enum: MADV_CORE 42 | 43 | enum: PROTECT 44 | enum: MADV_PROTECT 45 | 46 | # OpenBSD 47 | enum: SPACEAVAIL 48 | enum: MADV_SPACEAVAIL 49 | 50 | # Solaris 51 | enum: ACCESS_DEFAULT 52 | enum: ACCESS_LWP 53 | enum: ACCESS_MANY 54 | enum: ACCESS_MANY_PSET 55 | enum: PURGE 56 | enum: MADV_ACCESS_DEFAULT 57 | enum: MADV_ACCESS_LWP 58 | enum: MADV_ACCESS_MANY 59 | enum: MADV_ACCESS_MANY_PSET 60 | enum: MADV_PURGE 61 | -------------------------------------------------------------------------------- /tests/test_extra.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | import fmmap as mmap 6 | 7 | 8 | def test_off_by_one(): 9 | with mmap.mmap(-1, 10) as m: 10 | m.write(b"jj________") 11 | m.seek(0) 12 | assert m.find(b"jj") == 0 13 | assert m.rfind(b"jj") == 0 14 | 15 | 16 | def test_needle_too_big(): 17 | with mmap.mmap(-1, 10) as m: 18 | m.write(b"jj________") 19 | m.seek(0) 20 | assert m.find(b"abc", 9) == -1 21 | assert m.rfind(b"abc", 9) == -1 22 | 23 | 24 | def test_clean_namespace(): 25 | # We shouldn't soil the module namespace with our own extras 26 | assert getattr(mmap, "version_info", None) is None 27 | assert getattr(mmap, "kernel", None) is None 28 | assert getattr(mmap, "OS", None) is None 29 | assert getattr(mmap, "uname", None) is None 30 | 31 | 32 | @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only") 33 | def test_linux_constants(): 34 | assert getattr(mmap, "MADV_DONTFORK") > 0 35 | assert getattr(mmap, "MADV_REMOVE") > 0 36 | 37 | 38 | @pytest.mark.skipif(not sys.platform.startswith("freebsd"), reason="FreeBSD only") 39 | def test_freebsd_constants(): 40 | assert getattr(mmap, "MADV_FREE") > 0 41 | assert getattr(mmap, "MADV_NOSYNC") > 0 42 | 43 | 44 | @pytest.mark.skipif(not sys.platform.startswith("openbsd"), reason="OpenBSD only") 45 | def test_openbsd_constants(): 46 | assert getattr(mmap, "MADV_FREE") > 0 47 | assert getattr(mmap, "MADV_SPACEAVAIL") > 0 48 | 49 | 50 | @pytest.mark.skipif(not sys.platform.startswith("sunos"), reason="SunOS/Solaris only") 51 | def test_openbsd_constants(): 52 | assert getattr(mmap, "MADV_FREE") > 0 53 | assert getattr(mmap, "MADV_ACCESS_LWP") > 0 54 | -------------------------------------------------------------------------------- /benchmarks/grep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import mmap 4 | import sys 5 | import timeit 6 | 7 | import fmmap 8 | 9 | 10 | NUMBER=5 11 | 12 | 13 | needle = sys.argv[1].encode('utf-8') 14 | 15 | 16 | def find(m, needle=needle): 17 | return m.find(needle) 18 | 19 | 20 | def rfind(m, needle=needle): 21 | return m.rfind(needle) 22 | 23 | 24 | for name in sys.argv[2:]: 25 | with open(name, 'rb') as f: 26 | 27 | # built-in mmap 28 | with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m: 29 | a_find = find(m) 30 | print("mmap: find:", a_find) 31 | a1 = timeit.timeit('find(m)', number=NUMBER, globals={"m": m, "find": find}) 32 | print(f"{a1:.3f}") 33 | a_rfind = rfind(m) 34 | print("mmap: rfind:", a_rfind) 35 | a2 = timeit.timeit('find(m)', number=NUMBER, globals={"m": m, "find": rfind}) 36 | print(f"{a2:.3f}") 37 | 38 | # fmmap 39 | with fmmap.mmap(f.fileno(), 0, access=fmmap.ACCESS_READ) as m: 40 | b_find = find(m) 41 | print("fmmap: find:", a_find) 42 | b1 = timeit.timeit('find(m)', number=NUMBER, globals={"m": m, "find": find}) 43 | print(f"{b1:.3f}") 44 | b_rfind = rfind(m) 45 | print("fmmap: rfind:", b_rfind) 46 | b2 = timeit.timeit('find(m)', number=NUMBER, globals={"m": m, "find": rfind}) 47 | print(f"{b2:.3f}") 48 | 49 | if a_find != b_find: 50 | print("Results for find() differs!") 51 | if a_rfind != b_rfind: 52 | print("Results for rfind() differs!") 53 | 54 | print() 55 | print("Ratio mmap:fmmap (bigger than 1.0 means fmmap is faster)") 56 | print(f"find: {a1/b1:2.2f}") 57 | print(f"rfind: {a2/b2:2.2f}") 58 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Always prefer setuptools over distutils 2 | from setuptools import setup, find_packages 3 | from distutils.extension import Extension 4 | from os import path 5 | 6 | try: 7 | from Cython.Build import cythonize 8 | USE_CYTHON = True 9 | except ImportError: 10 | USE_CYTHON = False 11 | 12 | 13 | ext = '.pyx' if USE_CYTHON else '.c' 14 | extensions = [Extension("fmmap", ["src/fmmap"+ext], include_dirs=["src"])] 15 | 16 | if USE_CYTHON: 17 | from Cython.Build import cythonize 18 | extensions = cythonize(extensions) 19 | 20 | 21 | here = path.abspath(path.dirname(__file__)) 22 | 23 | # Get the long description from the README file 24 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 25 | long_description = f.read() 26 | 27 | 28 | setup( 29 | name='fmmap', 30 | version='2.0.0', 31 | description='A fast reimplementation of mmap', 32 | long_description=long_description, 33 | author='Friedel Wolff', 34 | author_email='friedel@translate.org.za', 35 | 36 | # For a list of valid classifiers, see https://pypi.org/classifiers/ 37 | classifiers=[ 38 | 'Development Status :: 5 - Production/Stable', 39 | 'Intended Audience :: Developers', 40 | 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 41 | 'Programming Language :: Cython', 42 | 'Programming Language :: Python :: 3.4', 43 | 'Programming Language :: Python :: 3.5', 44 | 'Programming Language :: Python :: 3.6', 45 | 'Programming Language :: Python :: 3.7', 46 | 'Programming Language :: Python :: 3.8', 47 | 'Programming Language :: Python :: 3.9', 48 | 'Operating System :: POSIX :: Linux', 49 | 'Operating System :: POSIX :: BSD :: FreeBSD', 50 | 'Operating System :: POSIX :: BSD :: NetBSD', 51 | 'Operating System :: POSIX :: BSD :: OpenBSD', 52 | 'Operating System :: POSIX :: SunOS/Solaris', 53 | 'Operating System :: MacOS :: MacOS X', 54 | 'Operating System :: Microsoft :: Windows', 55 | ], 56 | 57 | # Note that this is a string of words separated by whitespace, not a list. 58 | keywords='mmap', 59 | package_dir={'': 'src'}, 60 | ext_modules=extensions, 61 | 62 | # Specify which Python versions you support. In contrast to the 63 | # 'Programming Language' classifiers above, 'pip install' will check this 64 | # and refuse to install the project if the version does not match. If you 65 | # do not support Python 2, you can simplify this to '>=3.5' or similar, see 66 | # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires 67 | python_requires='>=3.4, <4', 68 | 69 | # List additional groups of dependencies here (e.g. development 70 | # dependencies). Users will be able to install these using the "extras" 71 | # syntax, for example: 72 | # 73 | # $ pip install sampleproject[dev] 74 | # 75 | # Similar to `install_requires` above, these must be valid existing 76 | # projects. 77 | extras_require={ # Optional 78 | 'dev': ['check-manifest'], 79 | 'test': ['coverage'], 80 | }, 81 | 82 | url='https://github.com/friedelwolff/fmmap/', 83 | # This field corresponds to the "Project-URL" metadata fields: 84 | # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use 85 | project_urls={ 86 | 'Bug Reports': 'https://github.com/friedelwolff/fmmap/issues', 87 | 'Source': 'https://github.com/friedelwolff/fmmap/', 88 | }, 89 | ) 90 | -------------------------------------------------------------------------------- /src/constants.h: -------------------------------------------------------------------------------- 1 | #ifndef _WIN32 2 | #include 3 | #define MEMMEM 1 4 | #define MEMRCHR 1 5 | 6 | #if defined(__FreeBSD__) && (__FreeBSD_version < 1200000) 7 | // memmem() on FreeBSD's libc is mostly slower than our local one 8 | #define MEMMEM 0 9 | #endif 10 | 11 | #if defined(__sun__) || defined(__sun) || defined(__MACH__) 12 | #define MEMRCHR 0 13 | #endif 14 | 15 | #else 16 | #define MEMMEM 0 17 | #define MEMRCHR 0 18 | #endif 19 | 20 | 21 | #if MEMMEM == 0 22 | // Just to keep the linker happy. We'll implement it with a different name. 23 | void * 24 | memmem(const void *big, size_t b_len, const void *little, size_t l_len) 25 | {} 26 | #endif 27 | 28 | #if MEMRCHR == 0 29 | // Just to keep the linker happy. We'll implement it with a different name. 30 | void * 31 | memrchr(const void *b, int c, size_t len) 32 | {} 33 | #endif 34 | 35 | 36 | 37 | 38 | // Linux: 39 | 40 | #ifdef MADV_HWPOISON 41 | #define HWPOISON 1 42 | #else 43 | #define HWPOISON 0 44 | #define MADV_HWPOISON 0 45 | #endif 46 | 47 | #ifdef MADV_MERGEABLE 48 | #define MERGEABLE 1 49 | #else 50 | #define MERGEABLE 0 51 | #define MADV_MERGEABLE 0 52 | #endif 53 | 54 | #ifdef MADV_SOFT_OFFLINE 55 | #define SOFT_OFFLINE 1 56 | #else 57 | #define SOFT_OFFLINE 0 58 | #define MADV_SOFT_OFFLINE 0 59 | #endif 60 | 61 | #ifdef MADV_HUGEPAGE 62 | #define HUGEPAGE 1 63 | #else 64 | #define HUGEPAGE 0 65 | #define MADV_HUGEPAGE 0 66 | #endif 67 | 68 | #ifdef MADV_DODUMP 69 | #define DUMP 1 70 | #else 71 | #define DUMP 0 72 | #define MADV_DODUMP 0 73 | #define MADV_DONTDUMP 0 74 | #endif 75 | 76 | #ifdef MADV_FREE 77 | #define FREE 1 78 | #else 79 | #define FREE 0 80 | #define MADV_FREE 0 81 | #endif 82 | 83 | #ifdef MADV_WIPEONFORK 84 | #define ONFORK 1 85 | #else 86 | #define ONFORK 0 87 | #define MADV_WIPEONFORK 0 88 | #define MADV_KEEPONFORK 0 89 | #endif 90 | 91 | 92 | // FreeBSD 93 | 94 | #ifdef MADV_NOSYNC 95 | #define NOSYNC 1 96 | #else 97 | #define NOSYNC 0 98 | #define MADV_NOSYNC 0 99 | #endif 100 | 101 | #ifdef MADV_AUTOSYNC 102 | #define AUTOSYNC 1 103 | #else 104 | #define AUTOSYNC 0 105 | #define MADV_AUTOSYNC 0 106 | #endif 107 | 108 | #ifdef MADV_CORE 109 | #define CORE 1 110 | #else 111 | #define CORE 0 112 | #define MADV_CORE 0 113 | #define MADV_NOCORE 0 114 | #endif 115 | 116 | #ifdef MADV_PROTECT 117 | #define PROTECT 1 118 | #else 119 | #define PROTECT 0 120 | #define MADV_PROTECT 0 121 | #endif 122 | 123 | 124 | // OpenBSD 125 | #ifdef MADV_SPACEAVAIL 126 | #define SPACEAVAIL 1 127 | #else 128 | #define SPACEAVAIL 0 129 | #define MADV_SPACEAVAIL 0 130 | #endif 131 | 132 | 133 | // Solaris and derivates 134 | 135 | #ifdef MADV_ACCESS_DEFAULT 136 | #define ACCESS_DEFAULT 1 137 | #else 138 | #define ACCESS_DEFAULT 0 139 | #define MADV_ACCESS_DEFAULT 0 140 | #endif 141 | 142 | #ifdef MADV_ACCESS_LWP 143 | #define ACCESS_LWP 1 144 | #else 145 | #define ACCESS_LWP 0 146 | #define MADV_ACCESS_LWP 0 147 | #endif 148 | 149 | #ifdef MADV_ACCESS_MANY 150 | #define ACCESS_MANY 1 151 | #else 152 | #define ACCESS_MANY 0 153 | #define MADV_ACCESS_MANY 0 154 | #endif 155 | 156 | #ifdef MADV_ACCESS_MANY_PSET 157 | #define ACCESS_MANY_PSET 1 158 | #else 159 | #define ACCESS_MANY_PSET 0 160 | #define MADV_ACCESS_MANY_PSET 0 161 | #endif 162 | 163 | #ifdef MADV_PURGE 164 | #define PURGE 1 165 | #else 166 | #define PURGE 0 167 | #define MADV_PURGE 0 168 | #endif 169 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================================================================== 2 | fmmap -- A fast implementation of mmap 3 | =========================================================================== 4 | 5 | This module is a reimplementation of Python's builtin module mmap. It aims to 6 | provide better performance while being API compatible with the builtin module. 7 | Development tracks new Python versions, therefore this module is mostly usable 8 | as a backport to older Python versions -- consult the documentation about any 9 | changes to the mmap API in Python. You should be able to shadow the builtin 10 | module and forget about it. 11 | 12 | Install on the command line: 13 | 14 | .. code:: shell 15 | 16 | pip install --upgrade fmmap 17 | 18 | Import in Python under the name ``mmap``: 19 | 20 | .. code:: python 21 | 22 | import fmmap as mmap 23 | 24 | Memory mapping is a technique of providing access to a file by mapping it into 25 | the virtual address space of the process and letting the operating system 26 | handle the input and output instead of explicitly reading from or writing to 27 | the file. It can provide better performance over normal file access in some 28 | cases. The builtin mmap module in Python exposes this functionality, but some 29 | of the implementation is not as fast as possible. 30 | 31 | Summary of the project status: 32 | 33 | 34 | The ``find()`` and ``rfind()`` functions in fmmap should be faster than the 35 | version in the standard library. These two functions also release the global 36 | interpreter lock (GIL) while searching, which might provide some benefit if 37 | you have multi-threaded code. 38 | 39 | A number of features, bug fixes and API changes introduced in the standard 40 | library between Python 3.5 - Python 3.9 are supported in fmmap when running on 41 | older versions, notably: 42 | 43 | - The API of ``flush()`` works like Python > 3.7. 44 | - ``madvise()`` is implemented and most of the ``MADV_...`` constants are exposed. 45 | 46 | 47 | Requirements and Assumptions 48 | ---------------------------- 49 | 50 | The following requirements are supported and tested: 51 | 52 | - Python versions: 3.4, 3.5, 3.6, 3.7, 3.8. 53 | - Interpreters: CPython. 54 | - Operating systems: 55 | 56 | - Linux 57 | - BSD systems (FreeBSD, NetBSD, OpenBSD) 58 | - SunOS/Solaris (illumos/OpenIndiana) 59 | 60 | The following operating systems receive limited testing, but should work: 61 | 62 | - macOS 63 | - Windows 64 | 65 | To implement the searching functionality, fmmap makes use of functions in the C 66 | library. The performance characteristics therefore are platform and version 67 | dependent. Recent versions of glibc is known to be very good. Some 68 | characteristics of your data can also influence performance. The performance of 69 | fmmap should be better than the built-in mmap module in most cases. 70 | 71 | For non-Windows platforms fmmap currently assumes that your platform has an 72 | ``madvise(2)`` implementation and the header file . 73 | 74 | 75 | Credits and Resources 76 | --------------------- 77 | 78 | The code and tests in this project are based on the standard library's `mmap`_ 79 | module. Additional tests from the pypy project are also duplicated here which 80 | helped to identify a few bugs. Most functionality is just inherited from the 81 | current runtime. The rest is implemented in optimized Cython code. 82 | 83 | .. _mmap: https://docs.python.org/3/library/mmap.html 84 | 85 | Further reading on Wikipedia: 86 | 87 | - `The mmap(2) system call `__ 88 | - `Memory-mapped file `__ 89 | 90 | Contributing 91 | ------------ 92 | 93 | 1. Clone this repository (``git clone ...``) 94 | 2. Create a virtualenv 95 | 3. Install package dependencies: ``pip install --upgrade pytest tox`` 96 | 4. Install package in development mode: ``pip install -e .`` 97 | 5. Change some code 98 | 6. Generate the compiled module: ``cythonize src/fmmap.pyx`` 99 | 7. Run the tests: in the project root simply execute ``pytest``, and afterwards 100 | preferably ``tox`` to test the full test matrix. Consider installing as many 101 | supported interpreters as possible (having them in your ``PATH`` is often 102 | sufficient). 103 | 8. Submit a pull request and check for any errors reported by the Continuous 104 | Integration service. 105 | 106 | License 107 | ------- 108 | 109 | The MPL 2.0 License 110 | 111 | Copyright (c) 2020 `Friedel Wolff `_. 112 | -------------------------------------------------------------------------------- /src/fmmap.pyx: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # cython: language_level=3 6 | 7 | from mmap import * 8 | import sys 9 | 10 | 11 | cdef str platform = sys.platform 12 | 13 | cdef int py_version = sys.hexversion 14 | DEF PY39 = 0x030901f0 15 | DEF PY38 = 0x030801f0 16 | DEF PY37 = 0x030701f0 17 | DEF PY36 = 0x030601f0 18 | 19 | 20 | import cython 21 | from cpython cimport exc 22 | from libc cimport string 23 | 24 | cimport constants 25 | 26 | 27 | cdef extern from "": 28 | #GNU extension to glibc 29 | unsigned char *memmem(const void *haystack, size_t haystacklen, 30 | const void *needle, size_t needlelen) nogil 31 | 32 | 33 | # See man memmem(3) 34 | cdef unsigned char *my_memmem( 35 | const unsigned char *buf_p, 36 | size_t haystack_len, 37 | const unsigned char *needle, 38 | size_t needle_len, 39 | ) nogil: 40 | cdef unsigned char *c 41 | i = haystack_len - needle_len + 1 42 | c = string.memchr(buf_p, needle[0], i) 43 | while c: 44 | if string.memcmp(c, needle, needle_len) == 0: 45 | return c 46 | i = haystack_len - (c - buf_p) - needle_len 47 | c = string.memchr(c + 1, needle[0], i) 48 | 49 | return NULL 50 | 51 | 52 | # See man memchr(3) 53 | cdef unsigned char *my_memrchr( 54 | unsigned char *b, 55 | int c, 56 | size_t n, 57 | ) nogil: 58 | b += n - 1 59 | while n: 60 | if c == b[0]: 61 | return b 62 | b -= 1 63 | n -= 1 64 | 65 | return NULL 66 | 67 | 68 | _transform_flush_return_value = lambda value: value 69 | 70 | 71 | if py_version < PY38: 72 | # We want to implement the return convention for flush() introduced in 73 | # Python 3.8. If we are running on an earlier version, let's massage the 74 | # return value: 75 | 76 | IF UNAME_SYSNAME == "Windows": 77 | def _transform_flush_return_value(value): 78 | if value == 0: 79 | # error 80 | exc.PyErr_SetFromWindowsErr(0) 81 | else: 82 | # success 83 | return None 84 | ELSE: 85 | def _transform_flush_return_value(value): 86 | # unix (and others) 87 | if value == 0: 88 | # success 89 | return None 90 | else: 91 | # error 92 | # Should not be reached, since flush() raises an exception on 93 | # errors on Python < 3.8 94 | pass 95 | 96 | 97 | IF UNAME_SYSNAME != "Windows": 98 | if py_version < PY38: 99 | # Constants needed for madvise 100 | 101 | from posix cimport mman 102 | 103 | MADV_NORMAL = mman.MADV_NORMAL 104 | MADV_RANDOM = mman.MADV_RANDOM 105 | MADV_SEQUENTIAL = mman.MADV_SEQUENTIAL 106 | MADV_WILLNEED = mman.MADV_WILLNEED 107 | MADV_DONTNEED = mman.MADV_DONTNEED 108 | # common in several Unix type systems; 109 | if constants.FREE: 110 | MADV_FREE = mman.MADV_FREE 111 | 112 | IF UNAME_SYSNAME == "Linux": 113 | 114 | from platform import uname 115 | 116 | kernel = tuple(int(x) for x in uname()[2].split('-')[0].split('.')) 117 | if kernel >= (2, 6, 16): 118 | MADV_REMOVE = mman.MADV_REMOVE 119 | MADV_DONTFORK = mman.MADV_DONTFORK 120 | MADV_DOFORK = mman.MADV_DOFORK 121 | if kernel >= (2, 6, 32): 122 | if constants.HWPOISON: 123 | MADV_HWPOISON = mman.MADV_HWPOISON 124 | if constants.MERGEABLE: 125 | MADV_MERGEABLE = mman.MADV_MERGEABLE 126 | MADV_UNMERGEABLE = mman.MADV_UNMERGEABLE 127 | if kernel >= (2, 6, 33) and constants.SOFT_OFFLINE: 128 | MADV_SOFT_OFFLINE = mman.MADV_SOFT_OFFLINE 129 | if kernel >= (2, 6, 38) and constants.HUGEPAGE: 130 | MADV_HUGEPAGE = mman.MADV_HUGEPAGE 131 | MADV_NOHUGEPAGE = mman.MADV_NOHUGEPAGE 132 | if kernel >= (3, 4, 0) and constants.DUMP: 133 | MADV_DONTDUMP = mman.MADV_DONTDUMP 134 | MADV_DODUMP = mman.MADV_DODUMP 135 | if kernel >= (4, 14, 0) and constants.ONFORK: 136 | MADV_WIPEONFORK = mman.MADV_WIPEONFORK 137 | MADV_KEEPONFORK = mman.MADV_KEEPONFORK 138 | del kernel 139 | del uname 140 | 141 | ELSE: 142 | # FreeBSD: 143 | if constants.NOSYNC: 144 | MADV_NOSYNC = constants.MADV_NOSYNC 145 | MADV_AUTOSYNC = constants.MADV_AUTOSYNC 146 | if constants.CORE: 147 | MADV_NOCORE = constants.MADV_NOCORE 148 | MADV_CORE = constants.MADV_CORE 149 | if constants.PROTECT: 150 | MADV_PROTECT = constants.MADV_PROTECT 151 | 152 | 153 | # Some madvise constants aren't in the standard library (in any Python 154 | # version so far), so we expose them here unconditionally if they are in 155 | # . 156 | 157 | # OpenBSD: 158 | if constants.SPACEAVAIL: 159 | MADV_SPACEAVAIL = constants.MADV_SPACEAVAIL 160 | 161 | # Solaris/illumos/OpenIndiana/SmartOs: 162 | if constants.ACCESS_DEFAULT: 163 | MADV_ACCESS_DEFAULT = constants.MADV_ACCESS_DEFAULT 164 | if constants.ACCESS_LWP: 165 | MADV_ACCESS_LWP = constants.MADV_ACCESS_LWP 166 | if constants.ACCESS_MANY: 167 | MADV_ACCESS_MANY = constants.MADV_ACCESS_MANY 168 | if constants.ACCESS_MANY_PSET: 169 | MADV_ACCESS_MANY_PSET = constants.MADV_ACCESS_MANY_PSET 170 | #illumos/OpenIndiana/SmartOS: 171 | if constants.PURGE: 172 | MADV_PURGE = constants.MADV_PURGE 173 | 174 | 175 | if py_version < PY37: 176 | ACCESS_DEFAULT = 0 177 | 178 | 179 | _mmap = mmap 180 | 181 | class mmap(_mmap): 182 | 183 | if py_version < PY39: 184 | 185 | def __init__(self, *args, **kwargs): 186 | self._fileno = kwargs.get("fileno", args[0]) 187 | # remember a few parameters for __repr__ 188 | self._access = kwargs.get("access", 0) # kwarg only 189 | self._offset = kwargs.get("offset", 0) # kwarg only 190 | _mmap.__init__(*args, **kwargs) 191 | 192 | def __repr__(self): 193 | if self.closed: 194 | return "" 195 | names = { 196 | ACCESS_DEFAULT: "ACCESS_DEFAULT", 197 | ACCESS_READ: "ACCESS_READ", 198 | ACCESS_WRITE: "ACCESS_WRITE", 199 | ACCESS_COPY: "ACCESS_COPY", 200 | } 201 | access = names.get(self._access, "unknown!") 202 | return ("" 208 | ) 209 | 210 | IF UNAME_SYSNAME != "Windows": 211 | if py_version < PY38: 212 | 213 | def madvise(self, option, start=0, length=None): 214 | cdef const unsigned char[:] buf = self 215 | cdef ssize_t buf_len = len(buf) 216 | cdef unsigned char *buf_p 217 | 218 | if length is None: 219 | length = buf_len 220 | 221 | if start < 0 or start >= buf_len: 222 | raise ValueError("madvise start out of bounds") 223 | if length < 0: 224 | raise ValueError("madvise length invalid") 225 | if sys.maxsize - start < length: 226 | raise OverflowError("madvise length too large") 227 | 228 | if start + length > buf_len: 229 | length = buf_len - start 230 | 231 | buf_p = &buf[start] 232 | if mman.madvise(buf_p, length, option) != 0: 233 | exc.PyErr_SetFromErrno(OSError) 234 | 235 | if py_version < PY38: 236 | 237 | def flush(self, *args, **kwargs): 238 | value = super().flush(*args, **kwargs) 239 | return _transform_flush_return_value(value) 240 | 241 | if py_version < PY36: 242 | 243 | def __add__(self, value): 244 | raise TypeError() 245 | 246 | def __mul__(self, value): 247 | raise TypeError() 248 | 249 | def write(self, bytes): 250 | cdef const unsigned char[:] buf = bytes 251 | super().write(buf) 252 | return len(buf) 253 | 254 | def resize(self, newsize): 255 | if self._access not in (ACCESS_WRITE, ACCESS_DEFAULT): 256 | raise TypeError() 257 | if newsize < 0 or sys.maxsize - newsize < self._offset: 258 | raise ValueError("new size out of range") 259 | if self._fileno != -1: 260 | super().resize(newsize) 261 | return 262 | 263 | # There is a bug in Python versions before 3.6. It would call 264 | # ftruncate(2) on file descriptor -1 (anonymous memory), so we 265 | # can't fall back on the built-in implementation. 266 | raise SystemError("Can't resize anonymous memory in Python < 3.6") 267 | 268 | 269 | def find(object self, sub, start=None, end=None): 270 | cdef const unsigned char[:] buf = self 271 | if start is None: 272 | start = self.tell() 273 | if end is None: 274 | end = len(buf) 275 | return self._find(sub, start, end) 276 | 277 | @cython.boundscheck(False) 278 | def _find(object self, r, ssize_t start, ssize_t end): 279 | cdef const unsigned char[:] buf = self 280 | cdef const unsigned char[:] needle = r 281 | cdef ssize_t buf_len = len(buf) 282 | cdef ssize_t needle_len = len(needle) 283 | cdef unsigned char *c 284 | cdef unsigned char *buf_p 285 | cdef unsigned char *needle_p 286 | 287 | # negative slicing and bounds checking 288 | if start < 0: 289 | start += buf_len 290 | if start < 0: 291 | start = 0 292 | elif start > buf_len: 293 | start = buf_len 294 | if end < 0: 295 | end += buf_len 296 | if end < 0: 297 | end = 0 298 | elif end > buf_len: 299 | end = buf_len 300 | 301 | # trivial cases 302 | if start >= end: 303 | return -1 304 | if needle_len == 0: 305 | return 0 306 | if buf_len == 0 or needle_len > buf_len: 307 | return -1 308 | if end - start < needle_len: 309 | return -1 310 | 311 | with nogil: 312 | buf_p = &buf[start] 313 | needle_p = &needle[0] 314 | if constants.MEMMEM: 315 | c = memmem(buf_p, end-start, needle_p, needle_len) 316 | else: 317 | c = my_memmem(buf_p, end-start, needle_p, needle_len) 318 | 319 | if c is NULL: 320 | return -1 321 | return c - buf_p + start 322 | 323 | def rfind(object self, sub, start=None, end=None): 324 | cdef const unsigned char[:] buf = self 325 | if start is None: 326 | start = self.tell() 327 | if end is None: 328 | end = len(buf) 329 | return self._rfind(sub, start, end) 330 | 331 | @cython.boundscheck(False) 332 | def _rfind(object self, r, ssize_t start, ssize_t end): 333 | cdef const unsigned char[:] buf = self 334 | cdef const unsigned char[:] needle = r 335 | cdef ssize_t buf_len = len(buf) 336 | cdef ssize_t needle_len = len(needle) 337 | cdef unsigned char *c = NULL 338 | cdef unsigned char *buf_p 339 | cdef unsigned char *needle_p 340 | cdef ssize_t i 341 | 342 | # negative slicing and bounds checking 343 | if start < 0: 344 | start += buf_len 345 | if start < 0: 346 | start = 0 347 | elif start > buf_len: 348 | start = buf_len 349 | if end < 0: 350 | end += buf_len 351 | if end < 0: 352 | end = 0 353 | elif end > buf_len: 354 | end = buf_len 355 | 356 | # trivial cases 357 | if start >= end: 358 | return -1 359 | if needle_len == 0: 360 | return 0 361 | if buf_len == 0 or needle_len > buf_len: 362 | return -1 363 | if needle_len > end - start: 364 | return -1 365 | 366 | with nogil: 367 | # Maybe not as fast as a good memmem(), but memrchr is hopefully 368 | # optimised. Worst case is still O(nm) where 369 | # - n = len(buf) 370 | # - m = len(needle) 371 | # Hopefully it is still faster than the naive algorithm in 372 | # CPython, or looping ourselves here. 373 | # 374 | # We repeatedly search for the first byte of needle from end to 375 | # start. When finding it, we check if the whole needle is there. 376 | 377 | buf_p = &buf[start] 378 | needle_p = &needle[0] 379 | i = end - start - needle_len + 1 380 | if constants.MEMRCHR: 381 | c = string.memrchr(buf_p, needle[0], i) 382 | while c: 383 | if string.memcmp(c, needle_p, needle_len) == 0: 384 | break 385 | c = string.memrchr(buf_p, needle[0], c - buf_p) 386 | 387 | else: 388 | c = my_memrchr(buf_p, needle[0], i) 389 | while c: 390 | if string.memcmp(c, needle_p, needle_len) == 0: 391 | break 392 | c = my_memrchr(buf_p, needle[0], c - buf_p) 393 | 394 | if c is NULL: 395 | return -1 396 | return c - buf_p + start 397 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /tests/test_mmap_pypy.py: -------------------------------------------------------------------------------- 1 | # From: 2 | # https://foss.heptapod.net/pypy/pypy/raw/branch/py3.6/pypy/module/mmap/test/test_mmap.py 3 | # Web view: 4 | # https://foss.heptapod.net/pypy/pypy/blob/branch/py3.6/pypy/module/mmap/test/test_mmap.py 5 | 6 | from __future__ import with_statement 7 | #from rpython.tool.udir import udir 8 | import os, sys, py 9 | import shutil 10 | 11 | import pytest 12 | from pytest import raises, skip 13 | 14 | 15 | import fmmap 16 | sys.modules["mmap"] = fmmap 17 | 18 | PLATFORMS_WITHOUT_MREMAP = ( 19 | "darwin", 20 | "freebsd", 21 | "openbsd", 22 | "sunos", 23 | ) 24 | 25 | class TestAppTestMMap: 26 | spaceconfig = dict(usemodules=('mmap',)) 27 | 28 | 29 | def setup_class(cls): 30 | shutil.rmtree("./tmp", ignore_errors=True) 31 | os.mkdir('./tmp') 32 | cls.tmpname = "./tmp/mmap-" 33 | 34 | def teardown_class(cls): 35 | try: 36 | shutil.rmtree("./tmp") 37 | except FileNotFoundError: 38 | pass 39 | 40 | def test_page_size(self): 41 | import mmap 42 | assert mmap.PAGESIZE > 0 43 | assert mmap.ALLOCATIONGRANULARITY > 0 44 | assert isinstance(mmap.PAGESIZE, int) 45 | assert isinstance(mmap.ALLOCATIONGRANULARITY, int) 46 | assert mmap.ALLOCATIONGRANULARITY % mmap.PAGESIZE == 0 47 | 48 | def test_attributes(self): 49 | import mmap 50 | import os 51 | assert isinstance(mmap.ACCESS_READ, int) 52 | assert isinstance(mmap.ACCESS_WRITE, int) 53 | assert isinstance(mmap.ACCESS_COPY, int) 54 | if os.name == "posix": 55 | assert isinstance(mmap.MAP_ANON, int) 56 | assert isinstance(mmap.MAP_ANONYMOUS, int) 57 | assert isinstance(mmap.MAP_PRIVATE, int) 58 | assert isinstance(mmap.MAP_SHARED, int) 59 | assert isinstance(mmap.PROT_EXEC, int) 60 | assert isinstance(mmap.PROT_READ, int) 61 | assert isinstance(mmap.PROT_WRITE, int) 62 | 63 | assert mmap.error is OSError 64 | 65 | def test_args(self): 66 | from mmap import mmap 67 | import os 68 | import sys 69 | 70 | raises(TypeError, mmap, "foo") 71 | raises(TypeError, mmap, 0, "foo") 72 | 73 | if os.name == "posix": 74 | raises(ValueError, mmap, 0, 1, 2, 3, 4) 75 | raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5) 76 | raises(TypeError, mmap, 0, 1, foo="foo") 77 | raises((TypeError, OverflowError), mmap, 0, -1) 78 | raises(OverflowError, mmap, 0, sys.maxsize ** 3) 79 | raises(ValueError, mmap, 0, 1, flags=2, access=3) 80 | raises(ValueError, mmap, 0, 1, access=123) 81 | elif os.name == "nt": 82 | raises(TypeError, mmap, 0, 1, 2, 3, 4) 83 | raises(TypeError, mmap, 0, 1, tagname=123) 84 | raises(TypeError, mmap, 0, 1, access="foo") 85 | raises(ValueError, mmap, 0, 1, access=-1) 86 | 87 | def test_subclass(self): 88 | import mmap 89 | class anon_mmap(mmap.mmap): 90 | def __new__(klass, *args, **kwargs): 91 | return mmap.mmap.__new__(klass, -1, *args, **kwargs) 92 | anon_mmap(mmap.PAGESIZE) 93 | 94 | def test_file_size(self): 95 | import os 96 | if os.name == "nt": 97 | skip("Only Unix checks file size") 98 | 99 | from mmap import mmap 100 | f = open(self.tmpname + "a", "wb+") 101 | 102 | f.write(b"c") 103 | f.flush() 104 | raises(ValueError, mmap, f.fileno(), 123) 105 | f.close() 106 | 107 | def test_create(self): 108 | from mmap import mmap 109 | f = open(self.tmpname + "b", "wb+") 110 | 111 | f.write(b"c") 112 | f.flush() 113 | m = mmap(f.fileno(), 1) 114 | assert m.read(99) == b"c" 115 | 116 | f.close() 117 | 118 | def test_close(self): 119 | from mmap import mmap 120 | f = open(self.tmpname + "c", "wb+") 121 | 122 | f.write(b"c") 123 | f.flush() 124 | m = mmap(f.fileno(), 1) 125 | m.close() 126 | raises(ValueError, m.read, 1) 127 | 128 | def test_read_byte(self): 129 | from mmap import mmap 130 | f = open(self.tmpname + "d", "wb+") 131 | 132 | f.write(b"c") 133 | f.flush() 134 | m = mmap(f.fileno(), 1) 135 | assert m.read_byte() == ord(b"c") 136 | raises(ValueError, m.read_byte) 137 | m.close() 138 | f.close() 139 | 140 | def test_readline(self): 141 | from mmap import mmap 142 | import os 143 | f = open(self.tmpname + "e", "wb+") 144 | 145 | f.write(b"foo\n") 146 | f.flush() 147 | m = mmap(f.fileno(), 4) 148 | assert m.readline() == b"foo\n" 149 | assert m.readline() == b"" 150 | m.close() 151 | f.close() 152 | 153 | def test_read(self): 154 | from mmap import mmap 155 | f = open(self.tmpname + "f", "wb+") 156 | 157 | f.write(b"foobar") 158 | f.flush() 159 | m = mmap(f.fileno(), 6) 160 | raises(TypeError, m.read, b"foo") 161 | assert m.read(1) == b"f" 162 | assert m.read(6) == b"oobar" 163 | assert m.read(1) == b"" 164 | m.close() 165 | f.close() 166 | 167 | def test_find(self): 168 | from mmap import mmap 169 | f = open(self.tmpname + "g", "wb+") 170 | 171 | f.write(b"foobar\0") 172 | f.flush() 173 | m = mmap(f.fileno(), 7) 174 | raises(TypeError, m.find, 123) 175 | raises(TypeError, m.find, b"foo", b"baz") 176 | assert m.find(b"b") == 3 177 | assert m.find(b"z") == -1 178 | assert m.find(b"o", 5) == -1 179 | assert m.find(b"ob") == 2 180 | assert m.find(b"\0") == 6 181 | assert m.find(b"ob", 1) == 2 182 | assert m.find(b"ob", 2) == 2 183 | assert m.find(b"ob", 3) == -1 184 | assert m.find(b"ob", -4) == -1 185 | assert m.find(b"ob", -5) == 2 186 | assert m.find(b"ob", -999999999) == 2 187 | assert m.find(b"ob", 1, 3) == -1 188 | assert m.find(b"ob", 1, 4) == 2 189 | assert m.find(b"ob", 1, 999999999) == 2 190 | assert m.find(b"ob", 1, 0) == -1 191 | assert m.find(b"ob", 1, -1) == 2 192 | assert m.find(b"ob", 1, -3) == 2 193 | assert m.find(b"ob", 1, -4) == -1 194 | # 195 | data = m.read(2) 196 | assert data == b"fo" 197 | assert m.find(b"o") == 2 198 | assert m.find(b"oo") == -1 199 | assert m.find(b"o", 0) == 1 200 | m.close() 201 | f.close() 202 | 203 | def test_rfind(self): 204 | from mmap import mmap 205 | f = open(self.tmpname + "g", "wb+") 206 | 207 | f.write(b"foobarfoobar\0") 208 | f.flush() 209 | m = mmap(f.fileno(), 13) 210 | raises(TypeError, m.rfind, 123) 211 | raises(TypeError, m.rfind, b"foo", b"baz") 212 | assert m.rfind(b"b") == 9 213 | assert m.rfind(b"z") == -1 214 | assert m.rfind(b"o", 11) == -1 215 | assert m.rfind(b"ob") == 8 216 | assert m.rfind(b"\0") == 12 217 | assert m.rfind(b"ob", 7) == 8 218 | assert m.rfind(b"ob", 8) == 8 219 | assert m.rfind(b"ob", 9) == -1 220 | assert m.rfind(b"ob", -4) == -1 221 | assert m.rfind(b"ob", -5) == 8 222 | assert m.rfind(b"ob", -999999999) == 8 223 | assert m.rfind(b"ob", 1, 3) == -1 224 | assert m.rfind(b"ob", 1, 4) == 2 225 | assert m.rfind(b"ob", 1, 999999999) == 8 226 | assert m.rfind(b"ob", 1, 0) == -1 227 | assert m.rfind(b"ob", 1, -1) == 8 228 | assert m.rfind(b"ob", 1, -3) == 8 229 | assert m.rfind(b"ob", 1, -4) == 2 230 | # 231 | data = m.read(8) 232 | assert data == b"foobarfo" 233 | assert m.rfind(b"o") == 8 234 | assert m.rfind(b"oo") == -1 235 | assert m.rfind(b"o", 0) == 8 236 | m.close() 237 | f.close() 238 | 239 | def test_is_modifiable(self): 240 | import mmap 241 | f = open(self.tmpname + "h", "wb+") 242 | 243 | f.write(b"foobar") 244 | f.flush() 245 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 246 | raises(TypeError, m.write, b'x') 247 | raises(TypeError, m.resize, 7) 248 | m.close() 249 | f.close() 250 | 251 | def test_seek(self): 252 | from mmap import mmap 253 | f = open(self.tmpname + "i", "wb+") 254 | 255 | f.write(b"foobar") 256 | f.flush() 257 | m = mmap(f.fileno(), 6) 258 | raises(TypeError, m.seek, b"foo") 259 | raises(TypeError, m.seek, 0, b"foo") 260 | raises(ValueError, m.seek, -1, 0) 261 | raises(ValueError, m.seek, -1, 1) 262 | raises(ValueError, m.seek, -7, 2) 263 | raises(ValueError, m.seek, 1, 3) 264 | raises(ValueError, m.seek, 10) 265 | m.seek(0) 266 | assert m.tell() == 0 267 | m.read(1) 268 | m.seek(1, 1) 269 | assert m.tell() == 2 270 | m.seek(0) 271 | m.seek(-1, 2) 272 | assert m.tell() == 5 273 | m.close() 274 | f.close() 275 | 276 | def test_write(self): 277 | import mmap 278 | f = open(self.tmpname + "j", "wb+") 279 | 280 | f.write(b"foobar") 281 | f.flush() 282 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 283 | raises(TypeError, m.write, b"foo") 284 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 285 | raises(TypeError, m.write, 123) 286 | raises(ValueError, m.write, b"c"*10) 287 | assert m.write(b"ciao\n") == 5 288 | m.seek(0) 289 | assert m.read(6) == b"ciao\nr" 290 | m.close() 291 | 292 | def test_write_byte(self): 293 | import mmap 294 | f = open(self.tmpname + "k", "wb+") 295 | 296 | f.write(b"foobar") 297 | f.flush() 298 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 299 | raises(TypeError, m.write_byte, b"f") 300 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 301 | raises(TypeError, m.write_byte, "a") 302 | raises(TypeError, m.write_byte, b"a") 303 | m.write_byte(ord("x")) 304 | m.seek(0) 305 | assert m.read(6) == b"xoobar" 306 | m.close() 307 | 308 | def test_size(self): 309 | from mmap import mmap 310 | f = open(self.tmpname + "l", "wb+") 311 | 312 | f.write(b"foobar") 313 | f.flush() 314 | m = mmap(f.fileno(), 5) 315 | assert m.size() == 6 # size of the underline file, not the mmap 316 | m.close() 317 | f.close() 318 | 319 | def test_tell(self): 320 | from mmap import mmap 321 | f = open(self.tmpname + "m", "wb+") 322 | 323 | f.write(b"c") 324 | f.flush() 325 | m = mmap(f.fileno(), 1) 326 | assert m.tell() >= 0 327 | m.close() 328 | f.close() 329 | 330 | def test_flush(self): 331 | from mmap import mmap 332 | f = open(self.tmpname + "n", "wb+") 333 | 334 | f.write(b"foobar") 335 | f.flush() 336 | m = mmap(f.fileno(), 6) 337 | raises(TypeError, m.flush, 1, 2, 3) 338 | raises(TypeError, m.flush, 1, b"a") 339 | raises(ValueError, m.flush, 0, 99) 340 | m.flush() # return value is a bit meaningless, platform-dependent 341 | m.close() 342 | f.close() 343 | 344 | def test_length_0_large_offset(self): 345 | import mmap 346 | 347 | with open(self.tmpname, "wb") as f: 348 | f.write(115699 * b'm') 349 | with open(self.tmpname, "w+b") as f: 350 | raises(ValueError, mmap.mmap, f.fileno(), 0, offset=2147418112) 351 | 352 | def test_move(self): 353 | import mmap 354 | f = open(self.tmpname + "o", "wb+") 355 | 356 | f.write(b"foobar") 357 | f.flush() 358 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 359 | raises(TypeError, m.move, 1) 360 | raises(TypeError, m.move, 1, b"foo", 2) 361 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 362 | raises(ValueError, m.move, 7, 1, 2) 363 | raises(ValueError, m.move, 1, 7, 2) 364 | m.move(1, 3, 3) 365 | assert m.read(6) == b"fbarar" 366 | m.seek(0) 367 | m.move(1, 3, 2) 368 | a = m.read(6) 369 | assert a == b"frarar" 370 | m.close() 371 | f.close() 372 | 373 | def test_resize(self): 374 | import sys 375 | for platform in PLATFORMS_WITHOUT_MREMAP: 376 | if platform in sys.platform: 377 | skip("resize does not work under some platforms") 378 | 379 | import mmap 380 | import os 381 | 382 | f = open(self.tmpname + "p", "wb+") 383 | f.write(b"foobar") 384 | f.flush() 385 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 386 | raises(TypeError, m.resize, 1) 387 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_COPY) 388 | raises(TypeError, m.resize, 1) 389 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 390 | f_size = os.fstat(f.fileno()).st_size 391 | assert m.size() == f_size == 6 392 | m.resize(10) 393 | f_size = os.fstat(f.fileno()).st_size 394 | assert m.size() == f_size == 10 395 | m.close() 396 | f.close() 397 | 398 | def test_resize_bsd(self): 399 | import sys 400 | for platform in PLATFORMS_WITHOUT_MREMAP: 401 | if platform in sys.platform: 402 | break 403 | else: 404 | skip("test different resize on some platforms") 405 | 406 | import mmap 407 | import os 408 | 409 | f = open(self.tmpname + "p", "wb+") 410 | f.write(b"foobar") 411 | f.flush() 412 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 413 | raises(TypeError, m.resize, 1) 414 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_COPY) 415 | raises(TypeError, m.resize, 1) 416 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 417 | f_size = os.fstat(f.fileno()).st_size 418 | assert m.size() == f_size == 6 419 | raises(SystemError, m.resize, 10) 420 | f_size = os.fstat(f.fileno()).st_size 421 | assert m.size() == f_size == 6 422 | 423 | def test_len(self): 424 | from mmap import mmap 425 | 426 | f = open(self.tmpname + "q", "wb+") 427 | f.write(b"foobar") 428 | f.flush() 429 | 430 | m = mmap(f.fileno(), 6) 431 | assert len(m) == 6 432 | m.close() 433 | f.close() 434 | 435 | def test_get_item(self): 436 | from mmap import mmap 437 | 438 | f = open(self.tmpname + "r", "wb+") 439 | f.write(b"foobar") 440 | f.flush() 441 | 442 | m = mmap(f.fileno(), 6) 443 | fn = lambda: m[b"foo"] 444 | raises(TypeError, fn) 445 | fn = lambda: m[-7] 446 | raises(IndexError, fn) 447 | assert m[0] == ord('f') 448 | assert m[-1] == ord('r') 449 | assert m[1::2] == b'obr' 450 | assert m[4:1:-2] == b'ao' 451 | m.close() 452 | f.close() 453 | 454 | def test_get_crash(self): 455 | import sys 456 | from mmap import mmap 457 | s = b'hallo!!!' 458 | m = mmap(-1, len(s)) 459 | m[:] = s 460 | assert m[1:None:sys.maxsize] == b'a' 461 | m.close() 462 | 463 | def test_set_item(self): 464 | import mmap 465 | 466 | f = open(self.tmpname + "s", "wb+") 467 | f.write(b"foobar") 468 | f.flush() 469 | 470 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) 471 | def fn(): m[1] = b'a' 472 | raises(TypeError, fn) 473 | m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_WRITE) 474 | def fn(): m[b"foo"] = b'a' 475 | raises(TypeError, fn) 476 | def fn(): m[-7] = b'a' 477 | raises(IndexError, fn) 478 | def fn(): m[1:3] = 'xx' 479 | raises((IndexError, TypeError), fn) # IndexError is in CPython, 480 | # but doesn't make much sense 481 | def fn(): m[1:4] = b"zz" 482 | raises((IndexError, ValueError), fn) 483 | def fn(): m[1:6] = b"z" * 6 484 | raises((IndexError, ValueError), fn) 485 | def fn(): m[:2] = b"z" * 5 486 | raises((IndexError, ValueError), fn) 487 | def fn(): m[0] = 256 488 | raises(ValueError, fn) 489 | m[1:3] = b'xx' 490 | assert m.read(6) == b"fxxbar" 491 | m[0] = ord('x') 492 | assert m[0] == ord('x') 493 | m[-6] = ord('y') 494 | m[3:6:2] = b'BR' 495 | m.seek(0) 496 | data = m.read(6) 497 | assert data == b"yxxBaR" 498 | m.close() 499 | f.close() 500 | 501 | def test_del_item(self): 502 | from mmap import mmap 503 | 504 | f = open(self.tmpname + "t", "wb+") 505 | f.write(b"foobar") 506 | f.flush() 507 | 508 | m = mmap(f.fileno(), 6) 509 | def fn(): del m[b"foo"] 510 | raises(TypeError, fn) 511 | def fn(): del m[1:3] 512 | raises(TypeError, fn) 513 | def fn(): del m[1] 514 | raises(TypeError, fn) 515 | m.close() 516 | f.close() 517 | 518 | def test_concatenation(self): 519 | from mmap import mmap 520 | 521 | f = open(self.tmpname + "u", "wb+") 522 | f.write(b"foobar") 523 | f.flush() 524 | 525 | m = mmap(f.fileno(), 6) 526 | def fn(): m + 1 527 | raises((SystemError, TypeError), fn) # SystemError is in CPython, 528 | def fn(m): m += 1 # but it doesn't make much 529 | raises((SystemError, TypeError), fn, m) # sense 530 | def fn(): 1 + m 531 | raises(TypeError, fn) 532 | m.close() 533 | f.close() 534 | 535 | def test_repeatition(self): 536 | from mmap import mmap 537 | 538 | f = open(self.tmpname + "v", "wb+") 539 | f.write(b"foobar") 540 | f.flush() 541 | 542 | m = mmap(f.fileno(), 6) 543 | def fn(): m * 1 544 | raises((SystemError, TypeError), fn) # SystemError is in CPython, 545 | def fn(m): m *= 1 # but it 546 | raises((SystemError, TypeError), fn, m) # doesn't 547 | def fn(): 1 * m # make much sense 548 | raises((SystemError, TypeError), fn) 549 | m.close() 550 | f.close() 551 | 552 | def test_slicing(self): 553 | from mmap import mmap 554 | 555 | f = open(self.tmpname + "v", "wb+") 556 | f.write(b"foobar") 557 | f.flush() 558 | 559 | f.seek(0) 560 | m = mmap(f.fileno(), 6) 561 | assert m[-3:7] == b"bar" 562 | 563 | assert m[1:0:1] == b"" 564 | 565 | f.close() 566 | 567 | def test_memoryview(self): 568 | from mmap import mmap, ACCESS_READ 569 | filename = self.tmpname + "y" 570 | f = open(filename, "bw+") 571 | f.write(b"foobar") 572 | f.flush() 573 | m = mmap(f.fileno(), 6) 574 | b = memoryview(m) 575 | assert len(b) == 6 576 | assert b.readonly is False 577 | assert b[3] == ord(b"b") 578 | assert b[:] == b"foobar" 579 | del b # For CPython: "exported pointers exist" 580 | m.close() 581 | f.close() 582 | with open(filename, "rb") as f: 583 | m = mmap(f.fileno(), 6, access=ACCESS_READ) 584 | b = memoryview(m) 585 | assert b.readonly is True 586 | assert b[:] == b"foobar" 587 | del b 588 | m.close() 589 | 590 | def test_offset(self): 591 | from mmap import mmap, ALLOCATIONGRANULARITY 592 | filename = self.tmpname + "y" 593 | f = open(filename, "wb+") 594 | f.write(b"foobar" * ALLOCATIONGRANULARITY) 595 | f.flush() 596 | size = ALLOCATIONGRANULARITY 597 | offset = 2 * ALLOCATIONGRANULARITY 598 | m = mmap(f.fileno(), size, offset=offset) 599 | assert m[:] == (b"foobar" * ALLOCATIONGRANULARITY)[offset:offset+size] 600 | assert len(m) == size 601 | m.close() 602 | f.close() 603 | 604 | def test_offset_more(self): 605 | from mmap import mmap, ALLOCATIONGRANULARITY 606 | 607 | with open(self.tmpname, "w+b") as f: 608 | halfsize = ALLOCATIONGRANULARITY 609 | f.write(b"\0" * halfsize) 610 | f.write(b"foo") 611 | f.write(b"\0" * (halfsize - 3)) 612 | f.flush() 613 | m = mmap(f.fileno(), 0) 614 | m.close() 615 | 616 | with open(self.tmpname, "r+b") as f: 617 | m = mmap(f.fileno(), halfsize, offset=halfsize) 618 | assert m[0:3] == b"foo" 619 | 620 | try: 621 | m.resize(512) 622 | except SystemError: 623 | pass 624 | else: 625 | assert len(m) == 512 626 | raises(ValueError, m.seek, 513, 0) 627 | assert m[0:3] == b"foo" 628 | with open(self.tmpname) as f: 629 | f.seek(0, 2) 630 | assert f.tell() == halfsize + 512 631 | assert m.size() == halfsize + 512 632 | m.close() 633 | 634 | def test_large_offset(self): 635 | import mmap 636 | import sys 637 | size = 0x14FFFFFFF 638 | if sys.platform.startswith('win') or sys.platform == 'darwin': 639 | skip('test requires %s bytes and a long time to run' % size) 640 | 641 | with open(self.tmpname, "w+b") as f: 642 | f.seek(size) 643 | f.write(b"A") 644 | f.flush() 645 | with open(self.tmpname, 'rb') as f2: 646 | f2.seek(size) 647 | c = f2.read(1) 648 | assert c == b'A' 649 | m = mmap.mmap(f2.fileno(), 0, offset=0x140000000, 650 | access=mmap.ACCESS_READ) 651 | try: 652 | assert m[0xFFFFFFF] == ord('A') 653 | finally: 654 | m.close() 655 | test_large_offset.is_large = True 656 | 657 | def test_large_filesize(self): 658 | import mmap 659 | import sys 660 | size = 0x17FFFFFFF 661 | if sys.platform.startswith('win') or sys.platform == 'darwin': 662 | skip('test requires %s bytes and a long time to run' % size) 663 | 664 | with open(self.tmpname, "w+b") as f: 665 | f.seek(size) 666 | f.write(b" ") 667 | f.flush() 668 | m = mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) 669 | try: 670 | assert m.size() == 0x180000000 671 | finally: 672 | m.close() 673 | test_large_filesize.is_large = True 674 | 675 | def test_context_manager(self): 676 | import mmap 677 | with mmap.mmap(-1, 10) as m: 678 | assert not m.closed 679 | assert m.closed 680 | 681 | def test_all(self): 682 | # this is a global test, ported from test_mmap.py 683 | import mmap 684 | from mmap import PAGESIZE 685 | import sys 686 | import os 687 | 688 | filename = self.tmpname + "w" 689 | 690 | f = open(filename, "wb+") 691 | 692 | # write 2 pages worth of data to the file 693 | f.write(b'\0' * PAGESIZE) 694 | f.write(b'foo') 695 | f.write(b'\0' * (PAGESIZE - 3)) 696 | f.flush() 697 | m = mmap.mmap(f.fileno(), 2 * PAGESIZE) 698 | f.close() 699 | 700 | # sanity checks 701 | assert m.find(b"foo") == PAGESIZE 702 | assert len(m) == 2 * PAGESIZE 703 | assert m[0] == 0 704 | assert m[0:3] == b'\0\0\0' 705 | 706 | # modify the file's content 707 | m[0] = ord('3') 708 | m[PAGESIZE+3:PAGESIZE+3+3] = b'bar' 709 | 710 | # check that the modification worked 711 | assert m[0] == ord('3') 712 | assert m[0:3] == b'3\0\0' 713 | assert m[PAGESIZE-1:PAGESIZE+7] == b'\0foobar\0' 714 | 715 | m.flush() 716 | 717 | # test seeking around 718 | m.seek(0,0) 719 | assert m.tell() == 0 720 | m.seek(42, 1) 721 | assert m.tell() == 42 722 | m.seek(0, 2) 723 | assert m.tell() == len(m) 724 | 725 | raises(ValueError, m.seek, -1) 726 | raises(ValueError, m.seek, 1, 2) 727 | raises(ValueError, m.seek, -len(m) - 1, 2) 728 | 729 | # try resizing map 730 | for platform in PLATFORMS_WITHOUT_MREMAP: 731 | if platform in sys.platform: 732 | break 733 | else: 734 | m.resize(512) 735 | 736 | assert len(m) == 512 737 | raises(ValueError, m.seek, 513, 0) 738 | 739 | # check that the underlying file is truncated too 740 | f = open(filename) 741 | f.seek(0, 2) 742 | assert f.tell() == 512 743 | f.close() 744 | assert m.size() == 512 745 | 746 | m.close() 747 | f.close() 748 | 749 | # test access=ACCESS_READ 750 | mapsize = 10 751 | f = open(filename, "wb") 752 | f.write(b"a" * mapsize) 753 | f.close() 754 | f = open(filename, "rb") 755 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ) 756 | assert m[:] == b'a' * mapsize 757 | def f(m): m[:] = b'b' * mapsize 758 | raises(TypeError, f, m) 759 | def fn(): m[0] = b'b' 760 | raises(TypeError, fn) 761 | def fn(m): m.seek(0, 0); m.write(b"abc") 762 | raises(TypeError, fn, m) 763 | def fn(m): m.seek(0, 0); m.write_byte(b"d") 764 | raises(TypeError, fn, m) 765 | if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)): 766 | raises(TypeError, m.resize, 2 * mapsize) 767 | assert open(filename, "rb").read() == b'a' * mapsize 768 | 769 | # opening with size too big 770 | f = open(filename, "r+b") 771 | if not os.name == "nt": 772 | # this should work under windows 773 | raises(ValueError, mmap.mmap, f.fileno(), mapsize + 1) 774 | f.close() 775 | 776 | # if _MS_WINDOWS: 777 | # # repair damage from the resizing test. 778 | # f = open(filename, 'r+b') 779 | # f.truncate(mapsize) 780 | # f.close() 781 | m.close() 782 | 783 | # test access=ACCESS_WRITE" 784 | f = open(filename, "r+b") 785 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE) 786 | m.write(b'c' * mapsize) 787 | m.seek(0) 788 | data = m.read(mapsize) 789 | assert data == b'c' * mapsize 790 | m.flush() 791 | m.close() 792 | f.close() 793 | f = open(filename, 'rb') 794 | stuff = f.read() 795 | f.close() 796 | assert stuff == b'c' * mapsize 797 | 798 | # test access=ACCESS_COPY 799 | f = open(filename, "r+b") 800 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY) 801 | m.write(b'd' * mapsize) 802 | m.seek(0) 803 | data = m.read(mapsize) 804 | assert data == b'd' * mapsize 805 | m.flush() 806 | assert open(filename, "rb").read() == b'c' * mapsize 807 | if not (("darwin" in sys.platform) or ("freebsd" in sys.platform)): 808 | raises(TypeError, m.resize, 2 * mapsize) 809 | m.close() 810 | f.close() 811 | 812 | # test invalid access 813 | f = open(filename, "r+b") 814 | raises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4) 815 | f.close() 816 | 817 | # test incompatible parameters 818 | if os.name == "posix": 819 | f = open(filename, "r+b") 820 | raises(ValueError, mmap.mmap, f.fileno(), mapsize, flags=mmap.MAP_PRIVATE, 821 | prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE) 822 | f.close() 823 | 824 | 825 | # bad file descriptor 826 | raises(EnvironmentError, mmap.mmap, -2, 4096) 827 | 828 | # do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, 829 | # searching for data with embedded \0 bytes didn't work. 830 | f = open(filename, 'wb+') 831 | data = b'aabaac\x00deef\x00\x00aa\x00' 832 | n = len(data) 833 | f.write(data) 834 | f.flush() 835 | m = mmap.mmap(f.fileno(), n) 836 | f.close() 837 | 838 | for start in range(n + 1): 839 | for finish in range(start, n + 1): 840 | sl = data[start:finish] 841 | assert m.find(sl) == data.find(sl) 842 | assert m.find(sl + b'x') == -1 843 | m.close() 844 | 845 | # test mapping of entire file by passing 0 for map length 846 | f = open(filename, "wb+") 847 | f.write(2**16 * b'm') 848 | f.close() 849 | f = open(filename, "rb+") 850 | m = mmap.mmap(f.fileno(), 0) 851 | assert len(m) == 2**16 852 | assert m.read(2**16) == 2**16 * b"m" 853 | m.close() 854 | f.close() 855 | 856 | # make move works everywhere (64-bit format problem earlier) 857 | f = open(filename, 'wb+') 858 | f.write(b"ABCDEabcde") 859 | f.flush() 860 | m = mmap.mmap(f.fileno(), 10) 861 | m.move(5, 0, 5) 862 | assert m.read(10) == b"ABCDEABCDE" 863 | m.close() 864 | f.close() 865 | 866 | def test_empty_file(self): 867 | import mmap 868 | f = open(self.tmpname, 'w+b') 869 | f.close() 870 | with open(self.tmpname, 'rb') as f: 871 | try: 872 | m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) 873 | m.close() 874 | assert False, "should not have been able to mmap empty file" 875 | except ValueError as e: 876 | assert str(e) == "cannot mmap an empty file" 877 | except BaseException as e: 878 | assert False, "unexpected exception: " + str(e) 879 | 880 | def test_read_all(self): 881 | from mmap import mmap 882 | f = open(self.tmpname + "f", "wb+") 883 | f.write(b"foobar") 884 | f.flush() 885 | 886 | m = mmap(f.fileno(), 6) 887 | assert m.read(None) == b"foobar" 888 | 889 | def test_resize_past_pos(self): 890 | import os, mmap, sys 891 | if os.name == "nt": 892 | skip("cannot resize anonymous mmaps on Windows") 893 | if sys.version_info < (2, 7, 13): 894 | skip("cannot resize anonymous mmaps before 2.7.13") 895 | m = mmap.mmap(-1, 8192) 896 | m.read(5000) 897 | try: 898 | m.resize(4096) 899 | except SystemError: 900 | skip("resizing not supported") 901 | assert m.tell() == 5000 902 | assert m.read(14) == b'' 903 | assert m.read(-1) == b'' 904 | raises(ValueError, m.read_byte) 905 | assert m.readline() == b'' 906 | raises(ValueError, m.write_byte, ord(b'b')) 907 | raises(ValueError, m.write, b'abc') 908 | assert m.tell() == 5000 909 | m.close() 910 | 911 | -------------------------------------------------------------------------------- /tests/test_mmap.py: -------------------------------------------------------------------------------- 1 | from test.support import (TESTFN, import_module, unlink, 2 | requires, _2G, _4G, gc_collect, cpython_only) 3 | import unittest 4 | import os 5 | import re 6 | import itertools 7 | import socket 8 | import sys 9 | import weakref 10 | 11 | import fmmap as mmap 12 | 13 | PAGESIZE = mmap.PAGESIZE 14 | 15 | 16 | class MmapTests(unittest.TestCase): 17 | 18 | def setUp(self): 19 | if os.path.exists(TESTFN): 20 | os.unlink(TESTFN) 21 | 22 | def tearDown(self): 23 | try: 24 | os.unlink(TESTFN) 25 | except OSError: 26 | pass 27 | 28 | def test_basic(self): 29 | # Test mmap module on Unix systems and Windows 30 | 31 | # Create a file to be mmap'ed. 32 | f = open(TESTFN, 'bw+') 33 | try: 34 | # Write 2 pages worth of data to the file 35 | f.write(b'\0'* PAGESIZE) 36 | f.write(b'foo') 37 | f.write(b'\0'* (PAGESIZE-3) ) 38 | f.flush() 39 | m = mmap.mmap(f.fileno(), 2 * PAGESIZE) 40 | finally: 41 | f.close() 42 | 43 | # Simple sanity checks 44 | 45 | tp = str(type(m)) # SF bug 128713: segfaulted on Linux 46 | self.assertEqual(m.find(b'foo'), PAGESIZE) 47 | 48 | self.assertEqual(len(m), 2*PAGESIZE) 49 | 50 | self.assertEqual(m[0], 0) 51 | self.assertEqual(m[0:3], b'\0\0\0') 52 | 53 | # Shouldn't crash on boundary (Issue #5292) 54 | self.assertRaises(IndexError, m.__getitem__, len(m)) 55 | self.assertRaises(IndexError, m.__setitem__, len(m), b'\0') 56 | 57 | # Modify the file's content 58 | m[0] = b'3'[0] 59 | m[PAGESIZE +3: PAGESIZE +3+3] = b'bar' 60 | 61 | # Check that the modification worked 62 | self.assertEqual(m[0], b'3'[0]) 63 | self.assertEqual(m[0:3], b'3\0\0') 64 | self.assertEqual(m[PAGESIZE-1 : PAGESIZE + 7], b'\0foobar\0') 65 | 66 | m.flush() 67 | 68 | # Test doing a regular expression match in an mmap'ed file 69 | match = re.search(b'[A-Za-z]+', m) 70 | if match is None: 71 | self.fail('regex match on mmap failed!') 72 | else: 73 | start, end = match.span(0) 74 | length = end - start 75 | 76 | self.assertEqual(start, PAGESIZE) 77 | self.assertEqual(end, PAGESIZE + 6) 78 | 79 | # test seeking around (try to overflow the seek implementation) 80 | m.seek(0,0) 81 | self.assertEqual(m.tell(), 0) 82 | m.seek(42,1) 83 | self.assertEqual(m.tell(), 42) 84 | m.seek(0,2) 85 | self.assertEqual(m.tell(), len(m)) 86 | 87 | # Try to seek to negative position... 88 | self.assertRaises(ValueError, m.seek, -1) 89 | 90 | # Try to seek beyond end of mmap... 91 | self.assertRaises(ValueError, m.seek, 1, 2) 92 | 93 | # Try to seek to negative position... 94 | self.assertRaises(ValueError, m.seek, -len(m)-1, 2) 95 | 96 | # Try resizing map 97 | try: 98 | m.resize(512) 99 | except SystemError: 100 | # resize() not supported 101 | # No messages are printed, since the output of this test suite 102 | # would then be different across platforms. 103 | pass 104 | else: 105 | # resize() is supported 106 | self.assertEqual(len(m), 512) 107 | # Check that we can no longer seek beyond the new size. 108 | self.assertRaises(ValueError, m.seek, 513, 0) 109 | 110 | # Check that the underlying file is truncated too 111 | # (bug #728515) 112 | f = open(TESTFN, 'rb') 113 | try: 114 | f.seek(0, 2) 115 | self.assertEqual(f.tell(), 512) 116 | finally: 117 | f.close() 118 | self.assertEqual(m.size(), 512) 119 | 120 | m.close() 121 | 122 | def test_access_parameter(self): 123 | # Test for "access" keyword parameter 124 | mapsize = 10 125 | with open(TESTFN, "wb") as fp: 126 | fp.write(b"a"*mapsize) 127 | with open(TESTFN, "rb") as f: 128 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ) 129 | self.assertEqual(m[:], b'a'*mapsize, "Readonly memory map data incorrect.") 130 | 131 | # Ensuring that readonly mmap can't be slice assigned 132 | try: 133 | m[:] = b'b'*mapsize 134 | except TypeError: 135 | pass 136 | else: 137 | self.fail("Able to write to readonly memory map") 138 | 139 | # Ensuring that readonly mmap can't be item assigned 140 | try: 141 | m[0] = b'b' 142 | except TypeError: 143 | pass 144 | else: 145 | self.fail("Able to write to readonly memory map") 146 | 147 | # Ensuring that readonly mmap can't be write() to 148 | try: 149 | m.seek(0,0) 150 | m.write(b'abc') 151 | except TypeError: 152 | pass 153 | else: 154 | self.fail("Able to write to readonly memory map") 155 | 156 | # Ensuring that readonly mmap can't be write_byte() to 157 | try: 158 | m.seek(0,0) 159 | m.write_byte(b'd') 160 | except TypeError: 161 | pass 162 | else: 163 | self.fail("Able to write to readonly memory map") 164 | 165 | # Ensuring that readonly mmap can't be resized 166 | try: 167 | m.resize(2*mapsize) 168 | except SystemError: # resize is not universally supported 169 | pass 170 | except TypeError: 171 | pass 172 | else: 173 | self.fail("Able to resize readonly memory map") 174 | with open(TESTFN, "rb") as fp: 175 | self.assertEqual(fp.read(), b'a'*mapsize, 176 | "Readonly memory map data file was modified") 177 | 178 | # Opening mmap with size too big 179 | with open(TESTFN, "r+b") as f: 180 | try: 181 | m = mmap.mmap(f.fileno(), mapsize+1) 182 | except ValueError: 183 | # we do not expect a ValueError on Windows 184 | # CAUTION: This also changes the size of the file on disk, and 185 | # later tests assume that the length hasn't changed. We need to 186 | # repair that. 187 | if sys.platform.startswith('win'): 188 | self.fail("Opening mmap with size+1 should work on Windows.") 189 | else: 190 | # we expect a ValueError on Unix, but not on Windows 191 | if not sys.platform.startswith('win'): 192 | self.fail("Opening mmap with size+1 should raise ValueError.") 193 | m.close() 194 | if sys.platform.startswith('win'): 195 | # Repair damage from the resizing test. 196 | with open(TESTFN, 'r+b') as f: 197 | f.truncate(mapsize) 198 | 199 | # Opening mmap with access=ACCESS_WRITE 200 | with open(TESTFN, "r+b") as f: 201 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE) 202 | # Modifying write-through memory map 203 | m[:] = b'c'*mapsize 204 | self.assertEqual(m[:], b'c'*mapsize, 205 | "Write-through memory map memory not updated properly.") 206 | m.flush() 207 | m.close() 208 | with open(TESTFN, 'rb') as f: 209 | stuff = f.read() 210 | self.assertEqual(stuff, b'c'*mapsize, 211 | "Write-through memory map data file not updated properly.") 212 | 213 | # Opening mmap with access=ACCESS_COPY 214 | with open(TESTFN, "r+b") as f: 215 | m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY) 216 | # Modifying copy-on-write memory map 217 | m[:] = b'd'*mapsize 218 | self.assertEqual(m[:], b'd' * mapsize, 219 | "Copy-on-write memory map data not written correctly.") 220 | m.flush() 221 | with open(TESTFN, "rb") as fp: 222 | self.assertEqual(fp.read(), b'c'*mapsize, 223 | "Copy-on-write test data file should not be modified.") 224 | # Ensuring copy-on-write maps cannot be resized 225 | self.assertRaises(TypeError, m.resize, 2*mapsize) 226 | m.close() 227 | 228 | # Ensuring invalid access parameter raises exception 229 | with open(TESTFN, "r+b") as f: 230 | self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4) 231 | 232 | if os.name == "posix": 233 | # Try incompatible flags, prot and access parameters. 234 | with open(TESTFN, "r+b") as f: 235 | self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize, 236 | flags=mmap.MAP_PRIVATE, 237 | prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE) 238 | 239 | # Try writing with PROT_EXEC and without PROT_WRITE 240 | prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0) 241 | with open(TESTFN, "r+b") as f: 242 | m = mmap.mmap(f.fileno(), mapsize, prot=prot) 243 | self.assertRaises(TypeError, m.write, b"abcdef") 244 | self.assertRaises(TypeError, m.write_byte, 0) 245 | m.close() 246 | 247 | def test_bad_file_desc(self): 248 | # Try opening a bad file descriptor... 249 | self.assertRaises(OSError, mmap.mmap, -2, 4096) 250 | 251 | def test_tougher_find(self): 252 | # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, 253 | # searching for data with embedded \0 bytes didn't work. 254 | with open(TESTFN, 'wb+') as f: 255 | 256 | data = b'aabaac\x00deef\x00\x00aa\x00' 257 | n = len(data) 258 | f.write(data) 259 | f.flush() 260 | m = mmap.mmap(f.fileno(), n) 261 | 262 | for start in range(n+1): 263 | for finish in range(start, n+1): 264 | slice = data[start : finish] 265 | self.assertEqual(m.find(slice), data.find(slice)) 266 | self.assertEqual(m.find(slice + b'x'), -1) 267 | m.close() 268 | 269 | def test_find_end(self): 270 | # test the new 'end' parameter works as expected 271 | with open(TESTFN, 'wb+') as f: 272 | data = b'one two ones' 273 | n = len(data) 274 | f.write(data) 275 | f.flush() 276 | m = mmap.mmap(f.fileno(), n) 277 | 278 | self.assertEqual(m.find(b'one'), 0) 279 | self.assertEqual(m.find(b'ones'), 8) 280 | self.assertEqual(m.find(b'one', 0, -1), 0) 281 | self.assertEqual(m.find(b'one', 1), 8) 282 | self.assertEqual(m.find(b'one', 1, -1), 8) 283 | self.assertEqual(m.find(b'one', 1, -2), -1) 284 | self.assertEqual(m.find(bytearray(b'one')), 0) 285 | 286 | 287 | def test_rfind(self): 288 | # test the new 'end' parameter works as expected 289 | with open(TESTFN, 'wb+') as f: 290 | data = b'one two ones' 291 | n = len(data) 292 | f.write(data) 293 | f.flush() 294 | m = mmap.mmap(f.fileno(), n) 295 | 296 | self.assertEqual(m.rfind(b'one'), 8) 297 | self.assertEqual(m.rfind(b'one '), 0) 298 | self.assertEqual(m.rfind(b'one', 0, -1), 8) 299 | self.assertEqual(m.rfind(b'one', 0, -2), 0) 300 | self.assertEqual(m.rfind(b'one', 1, -1), 8) 301 | self.assertEqual(m.rfind(b'one', 1, -2), -1) 302 | self.assertEqual(m.rfind(bytearray(b'one')), 8) 303 | 304 | 305 | def test_double_close(self): 306 | # make sure a double close doesn't crash on Solaris (Bug# 665913) 307 | with open(TESTFN, 'wb+') as f: 308 | f.write(2**16 * b'a') # Arbitrary character 309 | 310 | with open(TESTFN, 'rb') as f: 311 | mf = mmap.mmap(f.fileno(), 2**16, access=mmap.ACCESS_READ) 312 | mf.close() 313 | mf.close() 314 | 315 | def test_entire_file(self): 316 | # test mapping of entire file by passing 0 for map length 317 | with open(TESTFN, "wb+") as f: 318 | f.write(2**16 * b'm') # Arbitrary character 319 | 320 | with open(TESTFN, "rb+") as f, \ 321 | mmap.mmap(f.fileno(), 0) as mf: 322 | self.assertEqual(len(mf), 2**16, "Map size should equal file size.") 323 | self.assertEqual(mf.read(2**16), 2**16 * b"m") 324 | 325 | def test_length_0_offset(self): 326 | # Issue #10916: test mapping of remainder of file by passing 0 for 327 | # map length with an offset doesn't cause a segfault. 328 | # NOTE: allocation granularity is currently 65536 under Win64, 329 | # and therefore the minimum offset alignment. 330 | with open(TESTFN, "wb") as f: 331 | f.write((65536 * 2) * b'm') # Arbitrary character 332 | 333 | with open(TESTFN, "rb") as f: 334 | with mmap.mmap(f.fileno(), 0, offset=65536, access=mmap.ACCESS_READ) as mf: 335 | self.assertRaises(IndexError, mf.__getitem__, 80000) 336 | 337 | def test_length_0_large_offset(self): 338 | # Issue #10959: test mapping of a file by passing 0 for 339 | # map length with a large offset doesn't cause a segfault. 340 | with open(TESTFN, "wb") as f: 341 | f.write(115699 * b'm') # Arbitrary character 342 | 343 | with open(TESTFN, "w+b") as f: 344 | self.assertRaises(ValueError, mmap.mmap, f.fileno(), 0, 345 | offset=2147418112) 346 | 347 | def test_move(self): 348 | # make move works everywhere (64-bit format problem earlier) 349 | with open(TESTFN, 'wb+') as f: 350 | 351 | f.write(b"ABCDEabcde") # Arbitrary character 352 | f.flush() 353 | 354 | mf = mmap.mmap(f.fileno(), 10) 355 | mf.move(5, 0, 5) 356 | self.assertEqual(mf[:], b"ABCDEABCDE", "Map move should have duplicated front 5") 357 | mf.close() 358 | 359 | # more excessive test 360 | data = b"0123456789" 361 | for dest in range(len(data)): 362 | for src in range(len(data)): 363 | for count in range(len(data) - max(dest, src)): 364 | expected = data[:dest] + data[src:src+count] + data[dest+count:] 365 | m = mmap.mmap(-1, len(data)) 366 | m[:] = data 367 | m.move(dest, src, count) 368 | self.assertEqual(m[:], expected) 369 | m.close() 370 | 371 | # segfault test (Issue 5387) 372 | m = mmap.mmap(-1, 100) 373 | offsets = [-100, -1, 0, 1, 100] 374 | for source, dest, size in itertools.product(offsets, offsets, offsets): 375 | try: 376 | m.move(source, dest, size) 377 | except ValueError: 378 | pass 379 | 380 | offsets = [(-1, -1, -1), (-1, -1, 0), (-1, 0, -1), (0, -1, -1), 381 | (-1, 0, 0), (0, -1, 0), (0, 0, -1)] 382 | for source, dest, size in offsets: 383 | self.assertRaises(ValueError, m.move, source, dest, size) 384 | 385 | m.close() 386 | 387 | m = mmap.mmap(-1, 1) # single byte 388 | self.assertRaises(ValueError, m.move, 0, 0, 2) 389 | self.assertRaises(ValueError, m.move, 1, 0, 1) 390 | self.assertRaises(ValueError, m.move, 0, 1, 1) 391 | m.move(0, 0, 1) 392 | m.move(0, 0, 0) 393 | 394 | 395 | def test_anonymous(self): 396 | # anonymous mmap.mmap(-1, PAGE) 397 | m = mmap.mmap(-1, PAGESIZE) 398 | for x in range(PAGESIZE): 399 | self.assertEqual(m[x], 0, 400 | "anonymously mmap'ed contents should be zero") 401 | 402 | for x in range(PAGESIZE): 403 | b = x & 0xff 404 | m[x] = b 405 | self.assertEqual(m[x], b) 406 | 407 | def test_read_all(self): 408 | m = mmap.mmap(-1, 16) 409 | self.addCleanup(m.close) 410 | 411 | # With no parameters, or None or a negative argument, reads all 412 | m.write(bytes(range(16))) 413 | m.seek(0) 414 | self.assertEqual(m.read(), bytes(range(16))) 415 | m.seek(8) 416 | self.assertEqual(m.read(), bytes(range(8, 16))) 417 | m.seek(16) 418 | self.assertEqual(m.read(), b'') 419 | m.seek(3) 420 | self.assertEqual(m.read(None), bytes(range(3, 16))) 421 | m.seek(4) 422 | self.assertEqual(m.read(-1), bytes(range(4, 16))) 423 | m.seek(5) 424 | self.assertEqual(m.read(-2), bytes(range(5, 16))) 425 | m.seek(9) 426 | self.assertEqual(m.read(-42), bytes(range(9, 16))) 427 | 428 | def test_read_invalid_arg(self): 429 | m = mmap.mmap(-1, 16) 430 | self.addCleanup(m.close) 431 | 432 | self.assertRaises(TypeError, m.read, 'foo') 433 | self.assertRaises(TypeError, m.read, 5.5) 434 | self.assertRaises(TypeError, m.read, [1, 2, 3]) 435 | 436 | def test_extended_getslice(self): 437 | # Test extended slicing by comparing with list slicing. 438 | s = bytes(reversed(range(256))) 439 | m = mmap.mmap(-1, len(s)) 440 | m[:] = s 441 | self.assertEqual(m[:], s) 442 | indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300) 443 | for start in indices: 444 | for stop in indices: 445 | # Skip step 0 (invalid) 446 | for step in indices[1:]: 447 | self.assertEqual(m[start:stop:step], 448 | s[start:stop:step]) 449 | 450 | def test_extended_set_del_slice(self): 451 | # Test extended slicing by comparing with list slicing. 452 | s = bytes(reversed(range(256))) 453 | m = mmap.mmap(-1, len(s)) 454 | indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300) 455 | for start in indices: 456 | for stop in indices: 457 | # Skip invalid step 0 458 | for step in indices[1:]: 459 | m[:] = s 460 | self.assertEqual(m[:], s) 461 | L = list(s) 462 | # Make sure we have a slice of exactly the right length, 463 | # but with different data. 464 | data = L[start:stop:step] 465 | data = bytes(reversed(data)) 466 | L[start:stop:step] = data 467 | m[start:stop:step] = data 468 | self.assertEqual(m[:], bytes(L)) 469 | 470 | def make_mmap_file (self, f, halfsize): 471 | # Write 2 pages worth of data to the file 472 | f.write (b'\0' * halfsize) 473 | f.write (b'foo') 474 | f.write (b'\0' * (halfsize - 3)) 475 | f.flush () 476 | return mmap.mmap (f.fileno(), 0) 477 | 478 | def test_empty_file (self): 479 | f = open (TESTFN, 'w+b') 480 | f.close() 481 | with open(TESTFN, "rb") as f : 482 | self.assertRaisesRegex(ValueError, 483 | "cannot mmap an empty file", 484 | mmap.mmap, f.fileno(), 0, 485 | access=mmap.ACCESS_READ) 486 | 487 | def test_offset (self): 488 | f = open (TESTFN, 'w+b') 489 | 490 | try: # unlink TESTFN no matter what 491 | halfsize = mmap.ALLOCATIONGRANULARITY 492 | m = self.make_mmap_file (f, halfsize) 493 | m.close () 494 | f.close () 495 | 496 | mapsize = halfsize * 2 497 | # Try invalid offset 498 | f = open(TESTFN, "r+b") 499 | for offset in [-2, -1, None]: 500 | try: 501 | m = mmap.mmap(f.fileno(), mapsize, offset=offset) 502 | self.assertEqual(0, 1) 503 | except (ValueError, TypeError, OverflowError): 504 | pass 505 | else: 506 | self.assertEqual(0, 0) 507 | f.close() 508 | 509 | # Try valid offset, hopefully 8192 works on all OSes 510 | f = open(TESTFN, "r+b") 511 | m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize) 512 | self.assertEqual(m[0:3], b'foo') 513 | f.close() 514 | 515 | # Try resizing map 516 | try: 517 | m.resize(512) 518 | except SystemError: 519 | pass 520 | else: 521 | # resize() is supported 522 | self.assertEqual(len(m), 512) 523 | # Check that we can no longer seek beyond the new size. 524 | self.assertRaises(ValueError, m.seek, 513, 0) 525 | # Check that the content is not changed 526 | self.assertEqual(m[0:3], b'foo') 527 | 528 | # Check that the underlying file is truncated too 529 | f = open(TESTFN, 'rb') 530 | f.seek(0, 2) 531 | self.assertEqual(f.tell(), halfsize + 512) 532 | f.close() 533 | self.assertEqual(m.size(), halfsize + 512) 534 | 535 | m.close() 536 | 537 | finally: 538 | f.close() 539 | try: 540 | os.unlink(TESTFN) 541 | except OSError: 542 | pass 543 | 544 | def test_subclass(self): 545 | class anon_mmap(mmap.mmap): 546 | def __new__(klass, *args, **kwargs): 547 | return mmap.mmap.__new__(klass, -1, *args, **kwargs) 548 | anon_mmap(PAGESIZE) 549 | 550 | @unittest.skipUnless(hasattr(mmap, 'PROT_READ'), "needs mmap.PROT_READ") 551 | def test_prot_readonly(self): 552 | mapsize = 10 553 | with open(TESTFN, "wb") as fp: 554 | fp.write(b"a"*mapsize) 555 | with open(TESTFN, "rb") as f: 556 | m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ) 557 | self.assertRaises(TypeError, m.write, "foo") 558 | 559 | def test_error(self): 560 | self.assertIs(mmap.error, OSError) 561 | 562 | def test_io_methods(self): 563 | data = b"0123456789" 564 | with open(TESTFN, "wb") as fp: 565 | fp.write(b"x"*len(data)) 566 | with open(TESTFN, "r+b") as f: 567 | m = mmap.mmap(f.fileno(), len(data)) 568 | # Test write_byte() 569 | for i in range(len(data)): 570 | self.assertEqual(m.tell(), i) 571 | m.write_byte(data[i]) 572 | self.assertEqual(m.tell(), i+1) 573 | self.assertRaises(ValueError, m.write_byte, b"x"[0]) 574 | self.assertEqual(m[:], data) 575 | # Test read_byte() 576 | m.seek(0) 577 | for i in range(len(data)): 578 | self.assertEqual(m.tell(), i) 579 | self.assertEqual(m.read_byte(), data[i]) 580 | self.assertEqual(m.tell(), i+1) 581 | self.assertRaises(ValueError, m.read_byte) 582 | # Test read() 583 | m.seek(3) 584 | self.assertEqual(m.read(3), b"345") 585 | self.assertEqual(m.tell(), 6) 586 | # Test write() 587 | m.seek(3) 588 | m.write(b"bar") 589 | self.assertEqual(m.tell(), 6) 590 | self.assertEqual(m[:], b"012bar6789") 591 | m.write(bytearray(b"baz")) 592 | self.assertEqual(m.tell(), 9) 593 | self.assertEqual(m[:], b"012barbaz9") 594 | self.assertRaises(ValueError, m.write, b"ba") 595 | 596 | def test_non_ascii_byte(self): 597 | for b in (129, 200, 255): # > 128 598 | m = mmap.mmap(-1, 1) 599 | m.write_byte(b) 600 | self.assertEqual(m[0], b) 601 | m.seek(0) 602 | self.assertEqual(m.read_byte(), b) 603 | m.close() 604 | 605 | @unittest.skipUnless(os.name == 'nt', 'requires Windows') 606 | def test_tagname(self): 607 | data1 = b"0123456789" 608 | data2 = b"abcdefghij" 609 | assert len(data1) == len(data2) 610 | 611 | # Test same tag 612 | m1 = mmap.mmap(-1, len(data1), tagname="foo") 613 | m1[:] = data1 614 | m2 = mmap.mmap(-1, len(data2), tagname="foo") 615 | m2[:] = data2 616 | self.assertEqual(m1[:], data2) 617 | self.assertEqual(m2[:], data2) 618 | m2.close() 619 | m1.close() 620 | 621 | # Test different tag 622 | m1 = mmap.mmap(-1, len(data1), tagname="foo") 623 | m1[:] = data1 624 | m2 = mmap.mmap(-1, len(data2), tagname="boo") 625 | m2[:] = data2 626 | self.assertEqual(m1[:], data1) 627 | self.assertEqual(m2[:], data2) 628 | m2.close() 629 | m1.close() 630 | 631 | @cpython_only 632 | @unittest.skipUnless(os.name == 'nt', 'requires Windows') 633 | def test_sizeof(self): 634 | m1 = mmap.mmap(-1, 100) 635 | tagname = "foo" 636 | m2 = mmap.mmap(-1, 100, tagname=tagname) 637 | self.assertEqual(sys.getsizeof(m2), 638 | sys.getsizeof(m1) + len(tagname) + 1) 639 | 640 | @unittest.skipUnless(os.name == 'nt', 'requires Windows') 641 | def test_crasher_on_windows(self): 642 | # Should not crash (Issue 1733986) 643 | m = mmap.mmap(-1, 1000, tagname="foo") 644 | try: 645 | mmap.mmap(-1, 5000, tagname="foo")[:] # same tagname, but larger size 646 | except: 647 | pass 648 | m.close() 649 | 650 | # Should not crash (Issue 5385) 651 | with open(TESTFN, "wb") as fp: 652 | fp.write(b"x"*10) 653 | f = open(TESTFN, "r+b") 654 | m = mmap.mmap(f.fileno(), 0) 655 | f.close() 656 | try: 657 | m.resize(0) # will raise OSError 658 | except: 659 | pass 660 | try: 661 | m[:] 662 | except: 663 | pass 664 | m.close() 665 | 666 | @unittest.skipUnless(os.name == 'nt', 'requires Windows') 667 | def test_invalid_descriptor(self): 668 | # socket file descriptors are valid, but out of range 669 | # for _get_osfhandle, causing a crash when validating the 670 | # parameters to _get_osfhandle. 671 | s = socket.socket() 672 | try: 673 | with self.assertRaises(OSError): 674 | m = mmap.mmap(s.fileno(), 10) 675 | finally: 676 | s.close() 677 | 678 | def test_context_manager(self): 679 | with mmap.mmap(-1, 10) as m: 680 | self.assertFalse(m.closed) 681 | self.assertTrue(m.closed) 682 | 683 | def test_context_manager_exception(self): 684 | # Test that the OSError gets passed through 685 | with self.assertRaises(Exception) as exc: 686 | with mmap.mmap(-1, 10) as m: 687 | raise OSError 688 | self.assertIsInstance(exc.exception, OSError, 689 | "wrong exception raised in context manager") 690 | self.assertTrue(m.closed, "context manager failed") 691 | 692 | def test_weakref(self): 693 | # Check mmap objects are weakrefable 694 | mm = mmap.mmap(-1, 16) 695 | wr = weakref.ref(mm) 696 | self.assertIs(wr(), mm) 697 | del mm 698 | gc_collect() 699 | self.assertIs(wr(), None) 700 | 701 | def test_write_returning_the_number_of_bytes_written(self): 702 | mm = mmap.mmap(-1, 16) 703 | self.assertEqual(mm.write(b""), 0) 704 | self.assertEqual(mm.write(b"x"), 1) 705 | self.assertEqual(mm.write(b"yz"), 2) 706 | self.assertEqual(mm.write(b"python"), 6) 707 | 708 | @unittest.skipIf(os.name == 'nt', 'cannot resize anonymous mmaps on Windows') 709 | def test_resize_past_pos(self): 710 | m = mmap.mmap(-1, 8192) 711 | self.addCleanup(m.close) 712 | m.read(5000) 713 | try: 714 | m.resize(4096) 715 | except SystemError: 716 | self.skipTest("resizing not supported") 717 | self.assertEqual(m.read(14), b'') 718 | self.assertRaises(ValueError, m.read_byte) 719 | self.assertRaises(ValueError, m.write_byte, 42) 720 | self.assertRaises(ValueError, m.write, b'abc') 721 | 722 | def test_concat_repeat_exception(self): 723 | m = mmap.mmap(-1, 16) 724 | with self.assertRaises(TypeError): 725 | m + m 726 | with self.assertRaises(TypeError): 727 | m * 2 728 | 729 | def test_flush_return_value(self): 730 | # mm.flush() should return None on success, raise an 731 | # exception on error under all platforms. 732 | mm = mmap.mmap(-1, 16) 733 | self.addCleanup(mm.close) 734 | mm.write(b'python') 735 | result = mm.flush() 736 | self.assertIsNone(result) 737 | if sys.platform.startswith('linux'): 738 | # 'offset' must be a multiple of mmap.PAGESIZE on Linux. 739 | # See bpo-34754 for details. 740 | self.assertRaises(OSError, mm.flush, 1, len(b'python')) 741 | 742 | def test_repr(self): 743 | open_mmap_repr_pat = re.compile( 744 | r"\S+), " 746 | r"length=(?P\d+), " 747 | r"pos=(?P\d+), " 748 | r"offset=(?P\d+)>") 749 | closed_mmap_repr_pat = re.compile(r"") 750 | mapsizes = (50, 100, 1000, 1000000, 10000000) 751 | offsets = tuple((mapsize // 2 // mmap.ALLOCATIONGRANULARITY) 752 | * mmap.ALLOCATIONGRANULARITY for mapsize in mapsizes) 753 | for offset, mapsize in zip(offsets, mapsizes): 754 | data = b'a' * mapsize 755 | length = mapsize - offset 756 | accesses = ('ACCESS_DEFAULT', 'ACCESS_READ', 757 | 'ACCESS_COPY', 'ACCESS_WRITE') 758 | positions = (0, length//10, length//5, length//4) 759 | with open(TESTFN, "wb+") as fp: 760 | fp.write(data) 761 | fp.flush() 762 | for access, pos in itertools.product(accesses, positions): 763 | accint = getattr(mmap, access) 764 | with mmap.mmap(fp.fileno(), 765 | length, 766 | access=accint, 767 | offset=offset) as mm: 768 | mm.seek(pos) 769 | match = open_mmap_repr_pat.match(repr(mm)) 770 | self.assertIsNotNone(match, repr(mm)) 771 | self.assertEqual(match.group('access'), access) 772 | self.assertEqual(match.group('length'), str(length)) 773 | self.assertEqual(match.group('pos'), str(pos)) 774 | self.assertEqual(match.group('offset'), str(offset)) 775 | match = closed_mmap_repr_pat.match(repr(mm)) 776 | self.assertIsNotNone(match) 777 | 778 | @unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise') 779 | def test_madvise(self): 780 | size = 2 * PAGESIZE 781 | m = mmap.mmap(-1, size) 782 | 783 | with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): 784 | m.madvise(mmap.MADV_NORMAL, size) 785 | with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): 786 | m.madvise(mmap.MADV_NORMAL, -1) 787 | with self.assertRaisesRegex(ValueError, "madvise length invalid"): 788 | m.madvise(mmap.MADV_NORMAL, 0, -1) 789 | with self.assertRaisesRegex(OverflowError, "madvise length too large"): 790 | m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize) 791 | self.assertEqual(m.madvise(mmap.MADV_NORMAL), None) 792 | self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None) 793 | self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None) 794 | self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None) 795 | self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) 796 | 797 | 798 | class LargeMmapTests(unittest.TestCase): 799 | 800 | def setUp(self): 801 | unlink(TESTFN) 802 | 803 | def tearDown(self): 804 | unlink(TESTFN) 805 | 806 | def _make_test_file(self, num_zeroes, tail): 807 | if sys.platform[:3] == 'win' or sys.platform == 'darwin': 808 | requires('largefile', 809 | 'test requires %s bytes and a long time to run' % str(0x180000000)) 810 | f = open(TESTFN, 'w+b') 811 | try: 812 | f.seek(num_zeroes) 813 | f.write(tail) 814 | f.flush() 815 | except (OSError, OverflowError, ValueError): 816 | try: 817 | f.close() 818 | except (OSError, OverflowError): 819 | pass 820 | raise unittest.SkipTest("filesystem does not have largefile support") 821 | return f 822 | 823 | def test_large_offset(self): 824 | with self._make_test_file(0x14FFFFFFF, b" ") as f: 825 | with mmap.mmap(f.fileno(), 0, offset=0x140000000, access=mmap.ACCESS_READ) as m: 826 | self.assertEqual(m[0xFFFFFFF], 32) 827 | 828 | def test_large_filesize(self): 829 | with self._make_test_file(0x17FFFFFFF, b" ") as f: 830 | if sys.maxsize < 0x180000000: 831 | # On 32 bit platforms the file is larger than sys.maxsize so 832 | # mapping the whole file should fail -- Issue #16743 833 | with self.assertRaises(OverflowError): 834 | mmap.mmap(f.fileno(), 0x180000000, access=mmap.ACCESS_READ) 835 | with self.assertRaises(ValueError): 836 | mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) 837 | with mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) as m: 838 | self.assertEqual(m.size(), 0x180000000) 839 | 840 | # Issue 11277: mmap() with large (~4 GiB) sparse files crashes on OS X. 841 | 842 | def _test_around_boundary(self, boundary): 843 | tail = b' DEARdear ' 844 | start = boundary - len(tail) // 2 845 | end = start + len(tail) 846 | with self._make_test_file(start, tail) as f: 847 | with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m: 848 | self.assertEqual(m[start:end], tail) 849 | 850 | @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems") 851 | def test_around_2GB(self): 852 | self._test_around_boundary(_2G) 853 | 854 | @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems") 855 | def test_around_4GB(self): 856 | self._test_around_boundary(_4G) 857 | 858 | 859 | if __name__ == '__main__': 860 | unittest.main() 861 | --------------------------------------------------------------------------------