├── docs ├── _static │ └── .keepme ├── index.rst ├── make.bat ├── Makefile ├── api-reference.rst └── conf.py ├── .coveragerc ├── requirements.txt ├── dev-requirements.txt ├── .gitignore ├── setup.py ├── tasks.py ├── tox.ini ├── setup.cfg ├── .travis.yml ├── README.rst ├── bench └── bench_line_reading.py ├── test_zero_buffer.py └── zero_buffer.py /docs/_static/.keepme: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cffi >= 0.4 2 | six 3 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | tox 3 | flake8 4 | pytest 5 | coverage 6 | invoke 7 | twine 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | __pycache__ 3 | .tox/ 4 | *.egg-info/ 5 | _build/ 6 | *.egg 7 | ChangeLog 8 | AUTHORS 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | setuptools.setup( 5 | setup_requires=["pbr"], 6 | pbr=True 7 | ) 8 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | from invoke import task, run 2 | 3 | 4 | @task 5 | def release(version): 6 | """ 7 | Version should be a string like '0.4' or '1.0' 8 | """ 9 | run('git tag -s "{}"'.format(version)) 10 | run('python setup.py sdist') 11 | run('twine upload -s dist/zero_buffer-{}*'.format(version)) 12 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26,py27,py33,py34,py35,py36,pypy,pep8,docs 3 | 4 | [testenv] 5 | deps = 6 | pytest 7 | coverage 8 | commands = 9 | coverage run --source=zero_buffer -m pytest 10 | coverage report -m 11 | 12 | [testenv:pep8] 13 | deps = 14 | flake8 15 | commands = flake8 . 16 | 17 | [testenv:docs] 18 | deps = 19 | sphinx 20 | commands = 21 | sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html 22 | 23 | [flake8] 24 | exclude = .tox,*.egg 25 | select = E,W,F 26 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = zero_buffer 3 | summary = A fast, no-copy, buffer for Python 4 | description-file = 5 | README.rst 6 | author = Alex Gaynor and David Reid 7 | author-email = alex.gaynor@gmail.com 8 | home-page = https://github.com/alex/zero_buffer 9 | license = BSD 10 | ext_module = zero_buffer 11 | 12 | [files] 13 | modules = 14 | zero_buffer 15 | 16 | [build_sphinx] 17 | all_files = 1 18 | source-dir = docs 19 | 20 | [tool:pytest] 21 | norecursedirs = *.egg .tox 22 | 23 | [flake8] 24 | exclude = .tox/,docs/ 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | cache: 5 | directories: 6 | - $HOME/.cache/pip 7 | 8 | matrix: 9 | include: 10 | - python: 2.6 11 | env: TOXENV=py26 12 | - python: 2.7 13 | env: TOXENV=py27 14 | - python: 3.3 15 | env: TOXENV=py33 16 | - python: 3.4 17 | env: TOXENV=py34 18 | - python: 3.5 19 | env: TOXENV=py35 20 | - python: 3.6 21 | env: TOXENV=py36 22 | - python: pypy 23 | env: TOXENV=pypy 24 | - env: TOXENV=pep8 25 | - env: TOXENV=docs 26 | 27 | install: 28 | - pip install tox 29 | 30 | script: 31 | - tox 32 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | zero_buffer 2 | =========== 3 | 4 | .. image:: https://travis-ci.org/alex/zero_buffer.png?branch=master 5 | :target: https://travis-ci.org/alex/zero_buffer 6 | 7 | ``zero_buffer`` is a high-performance, zero-copy, implementation of a 8 | byte-buffer for Python. 9 | 10 | `Documentation`_ is available on ReadTheDocs. 11 | 12 | .. code-block:: python 13 | 14 | from zero_buffer import Buffer 15 | 16 | # Create a buffer which has space for 8192 bytes. 17 | b = Buffer.allocate(8192) 18 | with open(path, "rb") as f: 19 | # Read up to 8192 bytes from the file into the buffer 20 | b.read_from(f.fileno()) 21 | # Create a read-only view of the buffer, this performs no copying. 22 | view = b.view() 23 | # Split the view on colons, this returns a generator which yields sub-views 24 | # of the view. 25 | for part in view.split(b":"): 26 | print(part) 27 | 28 | ``zero_buffer`` works on Python 2.6, 2.7, 3.3+, and PyPy. 29 | 30 | .. _`Documentation`: https://zero-buffer.readthedocs.io/en/latest/ 31 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to Zero Buffer 2 | ====================== 3 | 4 | ``zero_buffer`` is a high-performance, zero-copy, implementation of a 5 | byte-buffer for Python. 6 | 7 | .. code-block:: python 8 | 9 | from zero_buffer import Buffer 10 | 11 | # Create a buffer which has space for 8192 bytes. 12 | b = Buffer.allocate(8192) 13 | with open(path, "rb") as f: 14 | # Read up to 8192 bytes from the file into the buffer 15 | b.read_from(f.fileno()) 16 | # Create a read-only view of the buffer, this performs no copying. 17 | view = b.view() 18 | # Split the view on colons, this returns a generator which yields sub-views 19 | # of the view. 20 | for part in view.split(b":"): 21 | print(part) 22 | 23 | ``zero_buffer`` works on Python 2.6, 2.7, 3.3+, and PyPy. 24 | 25 | Installation 26 | ------------ 27 | 28 | Install it with ``pip``: 29 | 30 | .. code-block:: console 31 | 32 | $ pip install zero_buffer 33 | 34 | If you are installing ``zero_buffer`` on Ubuntu, you may need to run the 35 | following before installing it with ``pip``. 36 | 37 | .. code-block:: console 38 | 39 | $ sudo apt-get install build-essential libffi-dev python-dev 40 | 41 | Contents 42 | -------- 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | 47 | api-reference 48 | -------------------------------------------------------------------------------- /bench/bench_line_reading.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from collections import defaultdict 3 | 4 | from six.moves import xrange 5 | 6 | from zero_buffer import Buffer, BufferCollator, BufferFull 7 | 8 | 9 | N = 100 10 | 11 | 12 | def run_py_bench(): 13 | d = defaultdict(int) 14 | with open("/usr/share/dict/words") as f: 15 | for line in f: 16 | d[line[0]] += 1 17 | 18 | 19 | def run_zero_buffer_bench(): 20 | d = defaultdict(int) 21 | cur_buffer = Buffer.allocate(8192) 22 | last_pos = 0 23 | collator = BufferCollator() 24 | with open("/usr/share/dict/words") as f: 25 | done = False 26 | while not done: 27 | try: 28 | read = cur_buffer.read_from(f.fileno()) 29 | except BufferFull: 30 | cur_buffer = Buffer.allocate(8192) 31 | last_pos = 0 32 | continue 33 | except EOFError: 34 | read = 0 35 | done = True 36 | view = cur_buffer.view(last_pos, last_pos + read) 37 | last_pos += read 38 | collator.append(view) 39 | if b"\n" in view: 40 | data = collator.collapse() 41 | last_newline_pos = data.rfind(b"\n") 42 | for line in data[:last_newline_pos].split(b"\n"): 43 | d[chr(line[0])] += 1 44 | collator.append(data[last_newline_pos + 1:]) 45 | 46 | 47 | def main(argv): 48 | name = argv[1] 49 | if name == "py": 50 | for i in xrange(N): 51 | run_py_bench() 52 | elif name == "zero_buffer": 53 | for i in xrange(N): 54 | run_zero_buffer_bench() 55 | else: 56 | raise SystemExit("argv[1] should be either py or zero_buffer") 57 | 58 | 59 | if __name__ == "__main__": 60 | main(sys.argv) 61 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\ZeroBuffer.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\ZeroBuffer.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ZeroBuffer.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ZeroBuffer.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ZeroBuffer" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ZeroBuffer" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/api-reference.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. currentmodule:: zero_buffer 5 | 6 | .. class:: Buffer 7 | 8 | A buffer is a fixed-size, append only, contigious region of memory. Once 9 | data is in it, that data cannot be mutated, however data can be read into 10 | the buffer with multiple calls. 11 | 12 | .. classmethod:: allocate(size) 13 | 14 | :param int size: Number of bytes. 15 | :return Buffer: The new buffer. 16 | 17 | Allocates a new buffer of ``size`` bytes. 18 | 19 | .. attribute:: capacity 20 | 21 | Returns the size of the underlying buffer. This is the same as what it 22 | was allocated with. 23 | 24 | .. attribute:: writepos 25 | 26 | Returns the current, internal writing position, this increases on calls 27 | to :meth:`read_from` and :meth:`add_bytes`. 28 | 29 | .. attribute:: free 30 | 31 | Returns the remaining space in the :class:`Buffer`. 32 | 33 | .. method:: read_from(fd) 34 | 35 | :param int fd: A file descriptor. 36 | :return int: Number of bytes read. 37 | :raises OSError: on an error reading from the file descriptor. 38 | :raises EOFError: when the read position of the file is at the end. 39 | :raises BufferFull: when the buffer has no remaining space when called 40 | 41 | Reads from the file descriptor into the :class:`Buffer`. Note that the 42 | number of bytes copied may be less than the number of bytes in the 43 | file. 44 | 45 | .. method:: add_bytes(b) 46 | 47 | :param bytes b: Bytes to copy into the buffer. 48 | :return int: Number of bytes copied into the buffer. 49 | :raises BufferFull: when the buffer has no remaining space when called 50 | 51 | Copies the bytes into the :class:`Buffer`. Note that the number of 52 | bytes copied may be less than ``len(b)`` if there isn't space in the 53 | :class:`Buffer`. 54 | 55 | .. method:: view(start=0, stop=None) 56 | 57 | :param int start: The byte-offset from the beggining of the buffer. 58 | :param int stop: The byte-offset from start. 59 | :return BufferView: 60 | :raises ValueError: If the stop is before the start, if the start is 61 | negative or after the writepos, or if the stop is 62 | after the writepos. 63 | 64 | Returns a view of the buffer's data. This does not perform any copying. 65 | 66 | 67 | .. class:: BufferView 68 | 69 | A buffer view is an immutable, fixed-size, view over a contigious region of 70 | memory. It exposes much of the same API as :class:`bytes`, except most 71 | methods return BufferViews and do not make copies of the data. A buffer 72 | view is either a view into a :class:`Buffer` or into another 73 | :class:`BufferView`. 74 | 75 | .. method:: __bytes__() 76 | 77 | Returns a copy of the contents of the view as a :class:`bytes`. 78 | 79 | .. method:: __len__() 80 | 81 | Returns the length of the view. 82 | 83 | .. method:: __eq__(other) 84 | 85 | Checks whether the contents of the view are equal to ``other``, which 86 | can be either a :class:`bytes` or a :class:`BufferView`. 87 | 88 | .. method:: __contains__(needle) 89 | 90 | Returns whether or not the ``needle`` exists in the view as a 91 | contigious series of bytes. 92 | 93 | .. method:: __getitem__(idx) 94 | 95 | If ``idx`` is a :class:`slice`, returns a :class:`BufferView` over that 96 | data, it does not perform a copy. If ``idx`` is an integer, it returns 97 | the ordinal value of the byte at that index. 98 | 99 | Unlike other containers in Python, this does not support slices with 100 | steps (``view[::2]``). 101 | 102 | .. method:: __add__(other) 103 | 104 | :param BufferView other: 105 | 106 | Returns a :class:`BufferView` over the concatenated contents. If 107 | ``other`` is contigious with ``self`` in memory, no copying is 108 | performed, otherwise both views are copied into a new one. 109 | 110 | .. method:: find(needle, start=0, stop=None) 111 | 112 | The same as :meth:`bytes.find`. 113 | 114 | .. method:: index(needle, start=0, stop=None) 115 | 116 | The same as :meth:`bytes.index`. 117 | 118 | .. method:: rfind(needle, start=0, stop=None) 119 | 120 | The same as :meth:`bytes.rfind`. 121 | 122 | .. method:: rindex(needle, start=0, stop=None) 123 | 124 | The same as :meth:`bytes.rindex`. 125 | 126 | .. method:: split(by, maxsplit=-1) 127 | 128 | Similar to :meth:`bytes.split`, except it returns an iterator (not a 129 | :class:`list`) over the results, and each result is a 130 | :class:`BufferView` (not a :class:`bytes`). 131 | 132 | .. method:: splitlines(keepends=False) 133 | 134 | Similar to :meth:`bytes.splitlines`, except it returns an iterator (not 135 | a :class:`list`) over the results, and each result is a 136 | :class:`BufferView` (not a :class:`bytes`). 137 | 138 | .. method:: isspace() 139 | 140 | The same as :meth:`bytes.isspace`. 141 | 142 | .. method:: isdigit() 143 | 144 | The same as :meth:`bytes.isdigit`. 145 | 146 | .. method:: isalpha() 147 | 148 | The same as :meth:`bytes.isalpha`. 149 | 150 | .. method:: strip(chars=None) 151 | 152 | The same as :meth:`bytes.strip` except it returns a :class:`BufferView` 153 | (and not a :class:`bytes`). 154 | 155 | .. method:: lstrip(chars=None) 156 | 157 | The same as :meth:`bytes.lstrip` except it returns a 158 | :class:`BufferView` (and not a :class:`bytes`). 159 | 160 | .. method:: rstrip(chars=None) 161 | 162 | The same as :meth:`bytes.rstrip` except it returns a 163 | :class:`BufferView` (and not a :class:`bytes`). 164 | 165 | .. method:: write_to(fd) 166 | 167 | :param int fd: A file descriptor. 168 | :return int: Number of bytes written. 169 | :raises OSError: on an error writing to the file descriptor. 170 | 171 | Writes the contents of the buffer to a file descriptor. Note that the 172 | number of bytes written may be less than the number of bytes in the 173 | buffer view. 174 | 175 | .. class:: BufferCollator 176 | 177 | A buffer collator is a collection of :class:`BufferView` objects which can 178 | be collapsed into a single :class:`BufferView`. 179 | 180 | .. method:: __len__() 181 | 182 | Returns the sum of the lengths of the views inside the collator. 183 | 184 | .. method:: append(view) 185 | 186 | :param BufferView view: 187 | 188 | Adds the contents of a view to the collator. 189 | 190 | .. method:: collapse() 191 | 192 | Collapses the contents of the collator into a single 193 | :class:`BufferView`. Also resets the internal state of the collator, so 194 | if you call it twice successively, the second call will return an empty 195 | :class:`BufferView`. 196 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Zero Buffer documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Oct 7 14:44:39 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | # -- General configuration ---------------------------------------------------- 15 | 16 | # If your documentation needs a minimal Sphinx version, state it here. 17 | # needs_sphinx = '1.0' 18 | 19 | # Add any Sphinx extension module names here, as strings. They can be 20 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 21 | extensions = ['sphinx.ext.intersphinx'] 22 | 23 | # Add any paths that contain templates here, relative to this directory. 24 | templates_path = ['_templates'] 25 | 26 | # The suffix of source filenames. 27 | source_suffix = '.rst' 28 | 29 | # The encoding of source files. 30 | # source_encoding = 'utf-8-sig' 31 | 32 | # The master toctree document. 33 | master_doc = 'index' 34 | 35 | # General information about the project. 36 | project = u'Zero Buffer' 37 | copyright = u'2013, Alex Gaynor and David Reid' 38 | 39 | # The language for content autogenerated by Sphinx. Refer to documentation 40 | # for a list of supported languages. 41 | # language = None 42 | 43 | # There are two options for replacing |today|: either, you set today to some 44 | # non-false value, then it is used: 45 | # today = '' 46 | # Else, today_fmt is used as the format for a strftime call. 47 | # today_fmt = '%B %d, %Y' 48 | 49 | # List of patterns, relative to source directory, that match files and 50 | # directories to ignore when looking for source files. 51 | exclude_patterns = ['_build'] 52 | 53 | # The reST default role (used for this markup: `text`) to use for all documents 54 | # default_role = None 55 | 56 | # If true, '()' will be appended to :func: etc. cross-reference text. 57 | # add_function_parentheses = True 58 | 59 | # If true, the current module name will be prepended to all description 60 | # unit titles (such as .. function::). 61 | # add_module_names = True 62 | 63 | # If true, sectionauthor and moduleauthor directives will be shown in the 64 | # output. They are ignored by default. 65 | # show_authors = False 66 | 67 | # The name of the Pygments (syntax highlighting) style to use. 68 | pygments_style = 'sphinx' 69 | 70 | # A list of ignored prefixes for module index sorting. 71 | # modindex_common_prefix = [] 72 | 73 | 74 | # -- Options for HTML output -------------------------------------------------- 75 | 76 | # The theme to use for HTML and HTML Help pages. See the documentation for 77 | # a list of builtin themes. 78 | html_theme = 'alabaster' 79 | 80 | # Theme options are theme-specific and customize the look and feel of a theme 81 | # further. For a list of options available for each theme, see the 82 | # documentation. 83 | # html_theme_options = {} 84 | 85 | # Add any paths that contain custom themes here, relative to this directory. 86 | # html_theme_path = [] 87 | 88 | # The name for this set of Sphinx documents. If None, it defaults to 89 | # " v documentation". 90 | # html_title = None 91 | 92 | # A shorter title for the navigation bar. Default is the same as html_title. 93 | # html_short_title = None 94 | 95 | # The name of an image file (relative to this directory) to place at the top 96 | # of the sidebar. 97 | # html_logo = None 98 | 99 | # The name of an image file (within the static path) to use as favicon of the 100 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 101 | # pixels large. 102 | # html_favicon = None 103 | 104 | # Add any paths that contain custom static files (such as style sheets) here, 105 | # relative to this directory. They are copied after the builtin static files, 106 | # so a file named "default.css" will overwrite the builtin "default.css". 107 | html_static_path = ['_static'] 108 | 109 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 110 | # using the given strftime format. 111 | # html_last_updated_fmt = '%b %d, %Y' 112 | 113 | # If true, SmartyPants will be used to convert quotes and dashes to 114 | # typographically correct entities. 115 | # html_use_smartypants = True 116 | 117 | # Custom sidebar templates, maps document names to template names. 118 | # html_sidebars = {} 119 | 120 | # Additional templates that should be rendered to pages, maps page names to 121 | # template names. 122 | # html_additional_pages = {} 123 | 124 | # If false, no module index is generated. 125 | # html_domain_indices = True 126 | 127 | # If false, no index is generated. 128 | # html_use_index = True 129 | 130 | # If true, the index is split into individual pages for each letter. 131 | # html_split_index = False 132 | 133 | # If true, links to the reST sources are added to the pages. 134 | # html_show_sourcelink = True 135 | 136 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 137 | # html_show_sphinx = True 138 | 139 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 140 | # html_show_copyright = True 141 | 142 | # If true, an OpenSearch description file will be output, and all pages will 143 | # contain a tag referring to it. The value of this option must be the 144 | # base URL from which the finished HTML is served. 145 | # html_use_opensearch = '' 146 | 147 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 148 | # html_file_suffix = None 149 | 150 | # Output file base name for HTML help builder. 151 | htmlhelp_basename = 'ZeroBufferdoc' 152 | 153 | 154 | # -- Options for LaTeX output ------------------------------------------------- 155 | 156 | latex_elements = { 157 | # The paper size ('letterpaper' or 'a4paper'). 158 | # 'papersize': 'letterpaper', 159 | 160 | # The font size ('10pt', '11pt' or '12pt'). 161 | # 'pointsize': '10pt', 162 | 163 | # Additional stuff for the LaTeX preamble. 164 | # 'preamble': '', 165 | } 166 | 167 | # Grouping the document tree into LaTeX files. List of tuples 168 | # (source start file, target name, title, author, documentclass [howto/manual]) 169 | latex_documents = [ 170 | ('index', 'ZeroBuffer.tex', u'Zero Buffer Documentation', 171 | u'Alex Gaynor and David Reid', 'manual'), 172 | ] 173 | 174 | # The name of an image file (relative to this directory) to place at the top of 175 | # the title page. 176 | # latex_logo = None 177 | 178 | # For "manual" documents, if this is true, then toplevel headings are parts, 179 | # not chapters. 180 | # latex_use_parts = False 181 | 182 | # If true, show page references after internal links. 183 | # latex_show_pagerefs = False 184 | 185 | # If true, show URL addresses after external links. 186 | # latex_show_urls = False 187 | 188 | # Documents to append as an appendix to all manuals. 189 | # latex_appendices = [] 190 | 191 | # If false, no module index is generated. 192 | # latex_domain_indices = True 193 | 194 | 195 | # -- Options for manual page output ------------------------------------------- 196 | 197 | # One entry per manual page. List of tuples 198 | # (source start file, name, description, authors, manual section). 199 | man_pages = [ 200 | ('index', 'zerobuffer', u'Zero Buffer Documentation', 201 | [u'Alex Gaynor and David Reid'], 1) 202 | ] 203 | 204 | # If true, show URL addresses after external links. 205 | # man_show_urls = False 206 | 207 | 208 | # -- Options for Texinfo output ----------------------------------------------- 209 | 210 | # Grouping the document tree into Texinfo files. List of tuples 211 | # (source start file, target name, title, author, 212 | # dir menu entry, description, category) 213 | texinfo_documents = [ 214 | ('index', 'ZeroBuffer', u'Zero Buffer Documentation', 215 | u'Alex Gaynor and David Reid', 'ZeroBuffer', 216 | 'A fast zero-copy byte buffer.', 'Miscellaneous'), 217 | ] 218 | 219 | # Documents to append as an appendix to all manuals. 220 | # texinfo_appendices = [] 221 | 222 | # If false, no module index is generated. 223 | # texinfo_domain_indices = True 224 | 225 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 226 | # texinfo_show_urls = 'footnote' 227 | 228 | 229 | # Example configuration for intersphinx: refer to the Python standard library. 230 | intersphinx_mapping = {'http://docs.python.org/': None} 231 | -------------------------------------------------------------------------------- /test_zero_buffer.py: -------------------------------------------------------------------------------- 1 | import errno 2 | 3 | import pytest 4 | 5 | from zero_buffer import Buffer, BufferView, BufferCollator, BufferFull 6 | 7 | 8 | @pytest.fixture 9 | def buf(): 10 | return Buffer.allocate(16) 11 | 12 | 13 | class TestBuffer(object): 14 | def test_read_from(self, buf, tmpdir): 15 | t = tmpdir.join("t.txt") 16 | t.write("abc123") 17 | with t.open() as f: 18 | res = buf.read_from(f.fileno()) 19 | assert res == 6 20 | assert buf.writepos == 6 21 | 22 | def test_read_from_error(self, buf): 23 | with pytest.raises(OSError) as exc_info: 24 | buf.read_from(-1) 25 | assert exc_info.value.errno == errno.EBADF 26 | 27 | def test_read_from_eof(self, buf, tmpdir): 28 | t = tmpdir.join("t.txt") 29 | t.write("") 30 | with t.open() as f: 31 | with pytest.raises(EOFError): 32 | buf.read_from(f.fileno()) 33 | 34 | def test_read_from_end_of_buffer(self, buf, tmpdir): 35 | t = tmpdir.join("t.txt") 36 | t.write("a" * 16) 37 | with t.open() as f: 38 | buf.read_from(f.fileno()) 39 | with t.open() as f: 40 | with pytest.raises(BufferFull): 41 | buf.read_from(f.fileno()) 42 | 43 | def test_free(self, buf, tmpdir): 44 | t = tmpdir.join("t.txt") 45 | t.write("abc") 46 | assert buf.free == 16 47 | with t.open() as f: 48 | buf.read_from(f.fileno()) 49 | assert buf.free == 13 50 | 51 | def test_capacity(self, buf): 52 | assert buf.capacity == 16 53 | buf.add_bytes(b"abc") 54 | assert buf.capacity == 16 55 | 56 | def test_add_bytes(self, buf): 57 | res = buf.add_bytes(b"abc") 58 | assert res == 3 59 | assert buf.writepos == 3 60 | 61 | def test_add_bytes_longer_than_buffer(self, buf): 62 | res = buf.add_bytes(b"a" * 20) 63 | assert res == 16 64 | assert buf.writepos == 16 65 | 66 | def test_add_bytes_buffer_full(self, buf): 67 | buf.add_bytes(b"a" * 16) 68 | with pytest.raises(BufferFull): 69 | buf.add_bytes(b"abc") 70 | 71 | def test_view(self, buf): 72 | buf.add_bytes(b"abc") 73 | view = buf.view(0, 3) 74 | assert isinstance(view, BufferView) 75 | view = buf.view() 76 | assert isinstance(view, BufferView) 77 | 78 | @pytest.mark.parametrize(("start", "stop"), [ 79 | (3, 0), 80 | (10, 11), 81 | (0, 11), 82 | (-2, 0), 83 | (0, -2), 84 | ]) 85 | def test_invalid_views(self, buf, start, stop): 86 | buf.add_bytes(b"abc123") 87 | with pytest.raises(ValueError): 88 | buf.view(start, stop) 89 | 90 | def test_writepos_assign(self, buf): 91 | with pytest.raises(AttributeError): 92 | buf.writepos = 12 93 | 94 | def test_repr(self, buf): 95 | buf.add_bytes(b"abc") 96 | assert repr(buf) == "Buffer(data=[97, 98, 99], capacity=16, free=13)" 97 | 98 | 99 | class TestBufferView(object): 100 | def test_bytes(self, buf): 101 | buf.add_bytes(b"abc") 102 | assert bytes(buf.view()) == b"abc" 103 | 104 | def test_repr(self, buf): 105 | buf.add_bytes(b"abc") 106 | assert repr(buf.view()) == "BufferView(data=[97, 98, 99])" 107 | 108 | def test_equality(self, buf): 109 | buf.add_bytes(b"abc") 110 | assert buf.view() == buf.view() 111 | assert buf.view() != buf.view(0, 2) 112 | assert not (buf.view() == buf.view(0, 2)) 113 | assert not (buf.view(0, 2) == buf.view(1, 3)) 114 | assert not (buf.view(0, 2) != buf.view(0, 2)) 115 | 116 | def test_equality_bytes(self, buf): 117 | buf.add_bytes(b"abc") 118 | assert buf.view() == b"abc" 119 | assert buf.view() != b"abd" 120 | assert not (buf.view() != b"abc") 121 | assert not (buf.view() == b"abd") 122 | assert buf.view() != b"ab" 123 | assert not (buf.view() == b"ab") 124 | 125 | def test_equality_other(self, buf): 126 | assert buf.view() != [] 127 | 128 | def test_contains(self, buf): 129 | buf.add_bytes(b"abc") 130 | view = buf.view() 131 | assert b"a" in view 132 | assert b"d" not in view 133 | 134 | def test_find_char(self, buf): 135 | buf.add_bytes(b"abc") 136 | view = buf.view() 137 | assert view.find(b"a") == 0 138 | assert view.find(b"c") == 2 139 | assert view.find(b"d") == -1 140 | 141 | def test_find_char_offsets(self, buf): 142 | buf.add_bytes(b"abcdefghijklm") 143 | view = buf.view() 144 | assert view.find(b"a", 1) == -1 145 | assert view.find(b"c", 2) == 2 146 | assert view.find(b"d", 2, 4) == 3 147 | assert view.find(b"e", 2, 3) == -1 148 | assert view.find(b"m", 0, 20) == 12 149 | assert view.find(b"a", -1) == 0 150 | assert view.find(b"a", 3, 2) == -1 151 | 152 | def test_find_empty_bytes(self, buf): 153 | buf.add_bytes(b"abc") 154 | view = buf.view() 155 | assert view.find(b"") == 0 156 | assert view.find(b"", 2) == 2 157 | 158 | def test_find_bytes(self, buf): 159 | buf.add_bytes(b"abc123") 160 | view = buf.view() 161 | assert view.find(b"cc") == -1 162 | assert view.find(b"ab") == 0 163 | assert view.find(b"c1") == 2 164 | buf.add_bytes(b"aabbcc") 165 | view = buf.view() 166 | assert view.find(b"aa") == 6 167 | assert view.find(b"abb") == 7 168 | 169 | def test_index(self, buf): 170 | buf.add_bytes(b"abc123") 171 | view = buf.view() 172 | assert view.index(b"b") == 1 173 | with pytest.raises(ValueError): 174 | view.index(b"d") 175 | 176 | def test_rfind_empty(self, buf): 177 | view = buf.view() 178 | assert view.rfind(b"") == 0 179 | 180 | def test_rfind_char(self, buf): 181 | buf.add_bytes(b"abc123") 182 | view = buf.view() 183 | assert view.rfind(b"c") == 2 184 | assert view.rfind(b"3") == 5 185 | assert view.rfind(b"4") == -1 186 | assert view.rfind(b"3", -2) == 5 187 | assert view.rfind(b"2", 0, 10) == 4 188 | assert view.rfind(b"2", 10, 0) == -1 189 | 190 | def test_rfind_bytes(self, buf): 191 | buf.add_bytes(b"123abc123") 192 | view = buf.view() 193 | assert view.rfind(b"cc") == -1 194 | assert view.rfind(b"23") == 7 195 | assert view.rfind(b"124") == -1 196 | 197 | def test_rindex(self, buf): 198 | buf.add_bytes(b"abc") 199 | view = buf.view() 200 | assert view.rindex(b"c") == 2 201 | with pytest.raises(ValueError): 202 | view.rindex(b"d") 203 | 204 | def test_subscript_slice(self, buf): 205 | buf.add_bytes(b"abc123") 206 | view = buf.view() 207 | assert view[:3] == b"abc" 208 | assert view[3:] == b"123" 209 | assert view[2:3] == b"c" 210 | with pytest.raises(ValueError): 211 | view[2:2:2] 212 | with pytest.raises(ValueError): 213 | view[3:2] 214 | 215 | def test_subscript(self, buf): 216 | buf.add_bytes(b"abc123") 217 | view = buf.view() 218 | assert view[0] == ord(b"a") 219 | assert view[-1] == ord(b"3") 220 | with pytest.raises(IndexError): 221 | view[7] 222 | with pytest.raises(IndexError): 223 | view[-7] 224 | 225 | def test_split_char(self, buf): 226 | buf.add_bytes(b"a-b-c") 227 | view = buf.view() 228 | assert list(view.split(b"-")) == [b"a", b"b", b"c"] 229 | 230 | def test_split_char_maxsplit(self, buf): 231 | buf.add_bytes(b"a-b-c") 232 | view = buf.view() 233 | assert list(view.split(b"-", 1)) == [b"a", b"b-c"] 234 | 235 | def test_split_empty(self, buf): 236 | view = buf.view() 237 | with pytest.raises(ValueError): 238 | view.split(b"") 239 | 240 | def test_split_bytes(self, buf): 241 | buf.add_bytes(b"a::b::c") 242 | view = buf.view() 243 | assert list(view.split(b"::")) == [b"a", b"b", b"c"] 244 | 245 | def test_split_bytes_maxsplit(self, buf): 246 | buf.add_bytes(b"a::b::c") 247 | view = buf.view() 248 | assert list(view.split(b"::", 1)) == [b"a", b"b::c"] 249 | 250 | def test_splitlines(self): 251 | b = Buffer.allocate(32) 252 | b.add_bytes(b"abc\ndef\n\rghi") 253 | assert list(b.view().splitlines()) == [b"abc", b"def", b"", b"ghi"] 254 | 255 | b = Buffer.allocate(32) 256 | b.add_bytes(b"abc\ndef\r\nghi") 257 | assert list(b.view().splitlines()) == [b"abc", b"def", b"ghi"] 258 | 259 | b = Buffer.allocate(32) 260 | b.add_bytes(b"\nabc\ndef\r\nghi\n\r") 261 | assert list(b.view().splitlines(True)) == [ 262 | b"\n", b"abc\n", b"def\r\n", b"ghi\n", b"\r" 263 | ] 264 | 265 | def test_strip_default_chars(self, buf): 266 | buf.add_bytes(b" \t\r\n\f\vabc\t\r\n\f\v ") 267 | view = buf.view() 268 | assert view.strip() == b"abc" 269 | 270 | def test_strip(self, buf): 271 | buf.add_bytes(b"abc123") 272 | view = buf.view() 273 | assert view.strip(b"ab3") == b"c12" 274 | 275 | def test_rstrip_default_chars(self, buf): 276 | buf.add_bytes(b" \t\r\n\f\vabc\t\r\n\f\v ") 277 | view = buf.view() 278 | assert view.rstrip() == b" \t\r\n\f\vabc" 279 | 280 | def test_lstrip_default_chars(self, buf): 281 | buf.add_bytes(b" \t\r\n\f\vabc\t\r\n\f\v ") 282 | view = buf.view() 283 | assert view.lstrip() == b"abc\t\r\n\f\v " 284 | 285 | def test_lstrip(self, buf): 286 | buf.add_bytes(b"abc123") 287 | view = buf.view() 288 | assert view.lstrip(b"ab3") == b"c123" 289 | 290 | def test_rstrip(self, buf): 291 | buf.add_bytes(b"abc123") 292 | view = buf.view() 293 | assert view.rstrip(b"ab3") == b"abc12" 294 | 295 | def test_isspace(self, buf): 296 | buf.add_bytes(b"a\t\r\n\f\v ") 297 | view = buf.view() 298 | assert not view[:1].isspace() 299 | assert view[1:].isspace() 300 | assert not buf.view(0, 0).isspace() 301 | assert not buf.view(0, 2).isspace() 302 | 303 | def test_iteration(self, buf): 304 | buf.add_bytes(b"abc") 305 | view = buf.view() 306 | assert list(view) == [ord(b"a"), ord(b"b"), ord(b"c")] 307 | 308 | def test_isdigit(self, buf): 309 | buf.add_bytes(b"123abc") 310 | assert not buf.view().isdigit() 311 | assert buf.view(0, 3).isdigit() 312 | assert not buf.view(0, 0).isdigit() 313 | 314 | def test_isalpha(self, buf): 315 | buf.add_bytes(b"abc123") 316 | assert not buf.view().isalpha() 317 | assert buf.view(0, 3).isalpha() 318 | assert not buf.view(0, 0).isalpha() 319 | 320 | def test_add_contigious(self, buf): 321 | buf.add_bytes(b"abc123") 322 | assert (buf.view(0, 3) + buf.view(3, 6)) == buf.view() 323 | 324 | def test_add_discontigious(self, buf): 325 | buf.add_bytes(b"abc123") 326 | bg = buf.view(0, 2) + buf.view(3, 6) 327 | assert bg == b"ab123" 328 | 329 | def test_add_typeerror(self, buf): 330 | view = buf.view() 331 | with pytest.raises(TypeError): 332 | view + 3 333 | 334 | def test_write_to(self, buf, tmpdir): 335 | buf.add_bytes(b"abcd") 336 | view = buf.view() 337 | with tmpdir.join("t.txt").open("wb") as f: 338 | res = view.write_to(f.fileno()) 339 | assert res == 4 340 | assert tmpdir.join("t.txt").read("rb") == b"abcd" 341 | 342 | def test_write_to_badfd(self, buf): 343 | buf.add_bytes(b"abcd") 344 | view = buf.view() 345 | with pytest.raises(OSError) as exc_info: 346 | view.write_to(-1) 347 | assert exc_info.value.errno == errno.EBADF 348 | 349 | def test_write_to_empty_bytes(self, buf, tmpdir): 350 | view = buf.view() 351 | with tmpdir.join("t.txt").open("wb") as f: 352 | res = view.write_to(f.fileno()) 353 | assert res == 0 354 | assert tmpdir.join("t.txt").read("rb") == b"" 355 | 356 | 357 | class TestBufferCollator(object): 358 | def test_single_item(self, buf): 359 | view = buf.view() 360 | collator = BufferCollator() 361 | collator.append(view) 362 | assert collator.collapse() is view 363 | 364 | def test_collapse_clears(self, buf): 365 | buf.add_bytes(b"abc") 366 | view = buf.view() 367 | collator = BufferCollator() 368 | collator.append(view) 369 | collator.collapse() 370 | view = collator.collapse() 371 | assert len(view) == 0 372 | 373 | def test_len(self, buf): 374 | buf.add_bytes(b"abc") 375 | view = buf.view() 376 | collator = BufferCollator() 377 | collator.append(view) 378 | collator.append(view) 379 | assert len(collator) == 6 380 | -------------------------------------------------------------------------------- /zero_buffer.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import six 4 | from six.moves import xrange 5 | 6 | import cffi 7 | 8 | 9 | _ffi = cffi.FFI() 10 | _ffi.cdef(""" 11 | ssize_t read(int, void *, size_t); 12 | ssize_t write(int, const void *, size_t); 13 | 14 | int memcmp(const void *, const void *, size_t); 15 | void *memchr(const void *, int, size_t); 16 | void *Zero_memrchr(const void *, int, size_t); 17 | void *memcpy(void *, const void *, size_t); 18 | """) 19 | _lib = _ffi.verify(""" 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #ifdef __GNU_SOURCE 26 | #define Zero_memrchr memrchr 27 | #else 28 | void *Zero_memrchr(const void *s, int c, size_t n) { 29 | const unsigned char *cp; 30 | if (n != 0) { 31 | cp = (unsigned char *)s + n; 32 | do { 33 | if (*(--cp) == (unsigned char)c) { 34 | return (void *)cp; 35 | } 36 | } while (--n != 0); 37 | } 38 | return NULL; 39 | } 40 | #endif 41 | """, extra_compile_args=["-D_GNU_SOURCE"]) 42 | 43 | BLOOM_WIDTH = _ffi.sizeof("long") * 8 44 | 45 | 46 | class BufferFull(Exception): 47 | pass 48 | 49 | 50 | class Buffer(object): 51 | def __init__(self, data, writepos): 52 | self._data = data 53 | self._writepos = writepos 54 | 55 | @classmethod 56 | def allocate(cls, size): 57 | return cls(_ffi.new("uint8_t[]", size), 0) 58 | 59 | def __repr__(self): 60 | return "Buffer(data=%r, capacity=%d, free=%d)" % ( 61 | [self._data[i] for i in xrange(self.writepos)], 62 | self.capacity, self.free 63 | ) 64 | 65 | @property 66 | def writepos(self): 67 | return self._writepos 68 | 69 | @property 70 | def capacity(self): 71 | return len(self._data) 72 | 73 | @property 74 | def free(self): 75 | return self.capacity - self.writepos 76 | 77 | def read_from(self, fd): 78 | if not self.free: 79 | raise BufferFull 80 | res = _lib.read(fd, self._data + self.writepos, self.free) 81 | if res == -1: 82 | raise OSError(_ffi.errno, os.strerror(_ffi.errno)) 83 | elif res == 0: 84 | raise EOFError 85 | self._writepos += res 86 | return res 87 | 88 | def add_bytes(self, b): 89 | if not self.free: 90 | raise BufferFull 91 | bytes_written = min(len(b), self.free) 92 | for i in xrange(bytes_written): 93 | self._data[self.writepos] = six.indexbytes(b, i) 94 | self._writepos += 1 95 | return bytes_written 96 | 97 | def view(self, start=0, stop=None): 98 | if stop is None: 99 | stop = self.writepos 100 | if stop < start: 101 | raise ValueError("stop is less than start") 102 | if not (0 <= start <= self.writepos): 103 | raise ValueError( 104 | "The start is either negative or after the writepos" 105 | ) 106 | if stop > self.writepos: 107 | raise ValueError("stop is after the writepos") 108 | return BufferView(self, self._data, start, stop) 109 | 110 | 111 | class BufferView(object): 112 | def __init__(self, buf, data, start, stop): 113 | self._keepalive = buf 114 | self._data = data + start 115 | self._length = stop - start 116 | 117 | def __bytes__(self): 118 | return _ffi.buffer(self._data, self._length)[:] 119 | if six.PY2: 120 | __str__ = __bytes__ 121 | 122 | def __repr__(self): 123 | return "BufferView(data=%r)" % ( 124 | [self._data[i] for i in xrange(len(self))] 125 | ) 126 | 127 | def __len__(self): 128 | return self._length 129 | 130 | def __eq__(self, other): 131 | if len(self) != len(other): 132 | return False 133 | if isinstance(other, BufferView): 134 | return _lib.memcmp(self._data, other._data, len(self)) == 0 135 | elif isinstance(other, bytes): 136 | for i in xrange(len(self)): 137 | if self[i] != six.indexbytes(other, i): 138 | return False 139 | return True 140 | else: 141 | return NotImplemented 142 | 143 | def __ne__(self, other): 144 | return not (self == other) 145 | 146 | def __contains__(self, data): 147 | return self.find(data) != -1 148 | 149 | def __getitem__(self, idx): 150 | if isinstance(idx, slice): 151 | start, stop, step = idx.indices(len(self)) 152 | if step != 1: 153 | raise ValueError("Can't slice with non-1 step.") 154 | if start > stop: 155 | raise ValueError("Can't slice backwards.") 156 | return BufferView(self._keepalive, self._data, start, stop) 157 | else: 158 | if idx < 0: 159 | idx += len(self) 160 | if not (0 <= idx < len(self)): 161 | raise IndexError(idx) 162 | return self._data[idx] 163 | 164 | def __add__(self, other): 165 | if isinstance(other, BufferView): 166 | collator = BufferCollator() 167 | collator.append(self) 168 | collator.append(other) 169 | return collator.collapse() 170 | else: 171 | return NotImplemented 172 | 173 | def find(self, needle, start=0, stop=None): 174 | stop = stop or len(self) 175 | if start < 0: 176 | start = 0 177 | if stop > len(self): 178 | stop = len(self) 179 | if stop - start < 0: 180 | return -1 181 | 182 | if len(needle) == 0: 183 | return start 184 | elif len(needle) == 1: 185 | res = _lib.memchr(self._data + start, ord(needle), stop - start) 186 | if res == _ffi.NULL: 187 | return -1 188 | else: 189 | return _ffi.cast("uint8_t *", res) - self._data 190 | else: 191 | mask, skip = self._make_find_mask(needle) 192 | return self._multi_char_find(needle, start, stop, mask, skip) 193 | 194 | def index(self, needle, start=0, stop=None): 195 | idx = self.find(needle, start, stop) 196 | if idx == -1: 197 | raise ValueError("substring not found") 198 | return idx 199 | 200 | def rfind(self, needle, start=0, stop=None): 201 | stop = stop or len(self) 202 | if start < 0: 203 | start = 0 204 | if stop > len(self): 205 | stop = len(self) 206 | if stop - start < 0: 207 | return -1 208 | 209 | if len(needle) == 0: 210 | return start 211 | elif len(needle) == 1: 212 | res = _lib.Zero_memrchr( 213 | self._data + start, ord(needle), stop - start 214 | ) 215 | if res == _ffi.NULL: 216 | return -1 217 | else: 218 | return _ffi.cast("uint8_t *", res) - self._data 219 | else: 220 | mask, skip = self._make_rfind_mask(needle) 221 | return self._multi_char_rfind(needle, start, stop, mask, skip) 222 | 223 | def rindex(self, needle, start=0, stop=None): 224 | idx = self.rfind(needle, start, stop) 225 | if idx == -1: 226 | raise ValueError("substring not found") 227 | return idx 228 | 229 | def split(self, by, maxsplit=-1): 230 | if len(by) == 0: 231 | raise ValueError("empty separator") 232 | elif len(by) == 1: 233 | return self._split_char(by, maxsplit) 234 | else: 235 | return self._split_multi_char(by, maxsplit) 236 | 237 | def _split_char(self, by, maxsplit): 238 | start = 0 239 | while maxsplit != 0: 240 | next = self.find(by, start) 241 | if next == -1: 242 | break 243 | yield self[start:next] 244 | start = next + 1 245 | maxsplit -= 1 246 | yield self[start:] 247 | 248 | def _split_multi_char(self, by, maxsplit): 249 | start = 0 250 | mask, skip = self._make_find_mask(by) 251 | while maxsplit != 0: 252 | next = self._multi_char_find(by, start, len(self), mask, skip) 253 | if next < 0: 254 | break 255 | yield self[start:next] 256 | start = next + len(by) 257 | maxsplit -= 1 258 | yield self[start:] 259 | 260 | def _bloom_add(self, mask, c): 261 | return mask | (1 << (c & (BLOOM_WIDTH - 1))) 262 | 263 | def _bloom(self, mask, c): 264 | return mask & (1 << (c & (BLOOM_WIDTH - 1))) 265 | 266 | def _make_find_mask(self, needle): 267 | mlast = len(needle) - 1 268 | mask = 0 269 | skip = mlast - 1 270 | for i in xrange(mlast): 271 | mask = self._bloom_add(mask, six.indexbytes(needle, i)) 272 | if needle[i] == needle[mlast]: 273 | skip = mlast - i - 1 274 | mask = self._bloom_add(mask, six.indexbytes(needle, mlast)) 275 | return mask, skip 276 | 277 | def _multi_char_find(self, needle, start, stop, mask, skip): 278 | i = start - 1 279 | w = (stop - start) - len(needle) 280 | while i + 1 <= start + w: 281 | i += 1 282 | if self._data[i + len(needle) - 1] == six.indexbytes(needle, -1): 283 | for j in xrange(len(needle) - 1): 284 | if self._data[i + j] != six.indexbytes(needle, j): 285 | break 286 | else: 287 | return i 288 | if ( 289 | i + len(needle) < len(self) and 290 | not self._bloom(mask, self._data[i + len(needle)]) 291 | ): 292 | i += len(needle) 293 | else: 294 | i += skip 295 | else: 296 | if ( 297 | i + len(needle) < len(self) and 298 | not self._bloom(mask, self._data[i + len(needle)]) 299 | ): 300 | i += len(needle) 301 | return -1 302 | 303 | def _make_rfind_mask(self, needle): 304 | mask = self._bloom_add(0, six.indexbytes(needle, 0)) 305 | skip = len(needle) - 1 306 | for i in xrange(len(needle) - 1, 0, -1): 307 | mask = self._bloom_add(mask, six.indexbytes(needle, i)) 308 | if needle[i] == needle[0]: 309 | skip = i - 1 310 | return mask, skip 311 | 312 | def _multi_char_rfind(self, needle, start, stop, mask, skip): 313 | i = start + (stop - start - len(needle)) + 1 314 | while i - 1 >= start: 315 | i -= 1 316 | if self[i] == six.indexbytes(needle, 0): 317 | for j in xrange(len(needle) - 1, 0, -1): 318 | if self[i + j] != six.indexbytes(needle, j): 319 | break 320 | else: 321 | return i 322 | if i - 1 >= 0 and not self._bloom(mask, self[i - 1]): 323 | i -= len(needle) 324 | else: 325 | i -= skip 326 | else: 327 | if i - 1 >= 0 and not self._bloom(mask, self[i - 1]): 328 | i -= len(needle) 329 | return -1 330 | 331 | def splitlines(self, keepends=False): 332 | i = 0 333 | j = 0 334 | while j < len(self): 335 | while ( 336 | i < len(self) and 337 | self[i] != ord(b"\n") and self[i] != ord(b"\r") 338 | ): 339 | i += 1 340 | eol = i 341 | if i < len(self): 342 | if ( 343 | self[i] == ord(b"\r") and 344 | i + 1 < len(self) and self[i + 1] == ord(b"\n") 345 | ): 346 | i += 2 347 | else: 348 | i += 1 349 | if keepends: 350 | eol = i 351 | yield self[j:eol] 352 | j = i 353 | 354 | def isspace(self): 355 | if not self: 356 | return False 357 | for ch in self: 358 | if ch != 32 and not (9 <= ch <= 13): 359 | return False 360 | return True 361 | 362 | def isdigit(self): 363 | if not self: 364 | return False 365 | for ch in self: 366 | if not (ord("0") <= ch <= ord("9")): 367 | return False 368 | return True 369 | 370 | def isalpha(self): 371 | if not self: 372 | return False 373 | for ch in self: 374 | if not (65 <= ch <= 90 or 97 <= ch <= 122): 375 | return False 376 | return True 377 | 378 | def _strip_none(self, left, right): 379 | lpos = 0 380 | rpos = len(self) 381 | 382 | if left: 383 | while lpos < rpos and chr(self[lpos]).isspace(): 384 | lpos += 1 385 | 386 | if right: 387 | while rpos > lpos and chr(self[rpos - 1]).isspace(): 388 | rpos -= 1 389 | return self[lpos:rpos] 390 | 391 | def _strip_chars(self, chars, left, right): 392 | lpos = 0 393 | rpos = len(self) 394 | 395 | if left: 396 | while lpos < rpos and six.int2byte(self[lpos]) in chars: 397 | lpos += 1 398 | 399 | if right: 400 | while rpos > lpos and six.int2byte(self[rpos - 1]) in chars: 401 | rpos -= 1 402 | return self[lpos:rpos] 403 | 404 | def strip(self, chars=None): 405 | if chars is None: 406 | return self._strip_none(left=True, right=True) 407 | else: 408 | return self._strip_chars(chars, left=True, right=True) 409 | 410 | def lstrip(self, chars=None): 411 | if chars is None: 412 | return self._strip_none(left=True, right=False) 413 | else: 414 | return self._strip_chars(chars, left=True, right=False) 415 | 416 | def rstrip(self, chars=None): 417 | if chars is None: 418 | return self._strip_none(left=False, right=True) 419 | else: 420 | return self._strip_chars(chars, left=False, right=True) 421 | 422 | def write_to(self, fd): 423 | res = _lib.write(fd, self._data, self._length) 424 | if res == -1: 425 | raise OSError(_ffi.errno, os.strerror(_ffi.errno)) 426 | return res 427 | 428 | 429 | class BufferCollator(object): 430 | def __init__(self): 431 | self._views = [] 432 | self._total_length = 0 433 | 434 | def __len__(self): 435 | return self._total_length 436 | 437 | def append(self, view): 438 | if self._views: 439 | last_view = self._views[-1] 440 | if ( 441 | last_view._keepalive is view._keepalive and 442 | last_view._data + len(last_view) == view._data 443 | ): 444 | self._views[-1] = BufferView( 445 | last_view._keepalive, 446 | last_view._data, 447 | 0, 448 | len(last_view) + len(view) 449 | ) 450 | else: 451 | self._views.append(view) 452 | else: 453 | self._views.append(view) 454 | self._total_length += len(view) 455 | 456 | def collapse(self): 457 | if len(self._views) == 1: 458 | result = self._views[0] 459 | else: 460 | data = _ffi.new("uint8_t[]", self._total_length) 461 | pos = 0 462 | for view in self._views: 463 | _lib.memcpy(data + pos, view._data, len(view)) 464 | pos += len(view) 465 | result = Buffer(data, self._total_length).view() 466 | del self._views[:] 467 | self._total_length = 0 468 | return result 469 | --------------------------------------------------------------------------------