├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── numpy_ringbuffer ├── __about__.py └── __init__.py ├── setup.py └── tests.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Test and publish to PyPI 2 | 3 | on: 4 | push: 5 | branches: 6 | pull_request: 7 | release: 8 | types: [ created ] 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10'] 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: 'Set up Python ${{matrix.python-version}}' 21 | uses: actions/setup-python@v3 22 | with: 23 | python-version: ${{matrix.python-version}} 24 | 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | 29 | - name: Install my package 30 | run: | 31 | python -m pip install . 32 | 33 | - name: Install test dependencies 34 | run: | 35 | pip install coverage 36 | 37 | - name: Test 38 | run: | 39 | coverage run tests.py 40 | 41 | - name: Clean up coverage data 42 | run: | 43 | # This is a workaround for the fact that the `[coverage:paths]` section 44 | # of `setup.cfg` is not actually applies until we run `combine`; so we 45 | # rename the report such that we can then "combine" it. 46 | mv .coverage .coverage.hack 47 | coverage combine 48 | coverage report 49 | 50 | - uses: codecov/codecov-action@v1 51 | 52 | deploy: 53 | needs: test 54 | runs-on: ubuntu-latest 55 | name: deploy 56 | steps: 57 | - uses: actions/checkout@v2 58 | - name: Set up Python '3.8' 59 | uses: actions/setup-python@v2 60 | with: 61 | python-version: '3.8' 62 | - name: "Install" 63 | run: | 64 | python -m pip install --upgrade pip 65 | python -m pip install build 66 | pip install setuptools wheel twine 67 | python setup.py sdist bdist_wheel 68 | 69 | - uses: actions/upload-artifact@v2 70 | with: 71 | name: dist 72 | path: dist 73 | 74 | # - name: Publish to Test PyPI (always) 75 | # uses: pypa/gh-action-pypi-publish@master 76 | # with: 77 | # user: __token__ 78 | # password: ${{ secrets.test_pypi_password }} 79 | # repository_url: https://test.pypi.org/legacy/ 80 | 81 | - name: Publish to PyPI (on tag) 82 | if: startsWith(github.ref, 'refs/tags/v') 83 | uses: pypa/gh-action-pypi-publish@release/v1 84 | with: 85 | user: __token__ 86 | password: ${{ secrets.pypi_token }} 87 | 88 | 89 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.egg-info 3 | /dist 4 | /build 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Eric Wieser 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # numpy_ringbuffer 2 | 3 | [![Build Status](https://github.com/eric-wieser/numpy_ringbuffer/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/eric-wieser/numpy_ringbuffer/actions/workflows/main.yml) 4 | [![codecov](https://codecov.io/gh/eric-wieser/numpy_ringbuffer/branch/master/graph/badge.svg)](https://codecov.io/gh/eric-wieser/numpy_ringbuffer) 5 | 6 | Ring (aka circular) buffers backed by a numpy array, supporting: 7 | 8 | * Operations from `collections.deque` 9 | * `b.append(val)` 10 | * `b.appendleft(val)` 11 | * `b.extend(val)` 12 | * `b.extendleft(val)` 13 | * `b.pop(val)` 14 | * `b.popleft(val)` 15 | * The `collections.Sequence` protocol (unoptimized) 16 | * C-side unwrapping into an array with `np.array(b)` 17 | * Arbitrary element dtypes, including extra dimensions like `RingBuffer(N, dtype=(int, 3))` 18 | 19 | For example: 20 | 21 | ```python 22 | import numpy as np 23 | from numpy_ringbuffer import RingBuffer 24 | 25 | r = RingBuffer(capacity=4, dtype=bool) 26 | r.append(True) 27 | r.appendleft(False) 28 | print(np.array(r)) # array([False, True]) 29 | ``` 30 | -------------------------------------------------------------------------------- /numpy_ringbuffer/__about__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.2" 2 | -------------------------------------------------------------------------------- /numpy_ringbuffer/__init__.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | try: 3 | from collections.abc import Sequence 4 | except ImportError: 5 | # Python < 3.3 6 | from collections import Sequence 7 | 8 | from .__about__ import * 9 | 10 | class RingBuffer(Sequence): 11 | def __init__(self, capacity, dtype=float, allow_overwrite=True): 12 | """ 13 | Create a new ring buffer with the given capacity and element type 14 | 15 | Parameters 16 | ---------- 17 | capacity: int 18 | The maximum capacity of the ring buffer 19 | dtype: data-type, optional 20 | Desired type of buffer elements. Use a type like (float, 2) to 21 | produce a buffer with shape (N, 2) 22 | allow_overwrite: bool 23 | If false, throw an IndexError when trying to append to an already 24 | full buffer 25 | """ 26 | self._arr = np.empty(capacity, dtype) 27 | self._left_index = 0 28 | self._right_index = 0 29 | self._capacity = capacity 30 | self._allow_overwrite = allow_overwrite 31 | 32 | def _unwrap(self): 33 | """ Copy the data from this buffer into unwrapped form """ 34 | return np.concatenate(( 35 | self._arr[self._left_index:min(self._right_index, self._capacity)], 36 | self._arr[:max(self._right_index - self._capacity, 0)] 37 | )) 38 | 39 | def _fix_indices(self): 40 | """ 41 | Enforce our invariant that 0 <= self._left_index < self._capacity 42 | """ 43 | if self._left_index >= self._capacity: 44 | self._left_index -= self._capacity 45 | self._right_index -= self._capacity 46 | elif self._left_index < 0: 47 | self._left_index += self._capacity 48 | self._right_index += self._capacity 49 | 50 | @property 51 | def is_full(self): 52 | """ True if there is no more space in the buffer """ 53 | return len(self) == self._capacity 54 | 55 | # numpy compatibility 56 | def __array__(self): 57 | return self._unwrap() 58 | 59 | @property 60 | def dtype(self): 61 | return self._arr.dtype 62 | 63 | @property 64 | def shape(self): 65 | return (len(self),) + self._arr.shape[1:] 66 | 67 | 68 | # these mirror methods from deque 69 | @property 70 | def maxlen(self): 71 | return self._capacity 72 | 73 | def append(self, value): 74 | if self.is_full: 75 | if not self._allow_overwrite: 76 | raise IndexError('append to a full RingBuffer with overwrite disabled') 77 | elif not len(self): 78 | return 79 | else: 80 | self._left_index += 1 81 | 82 | self._arr[self._right_index % self._capacity] = value 83 | self._right_index += 1 84 | self._fix_indices() 85 | 86 | def appendleft(self, value): 87 | if self.is_full: 88 | if not self._allow_overwrite: 89 | raise IndexError('append to a full RingBuffer with overwrite disabled') 90 | elif not len(self): 91 | return 92 | else: 93 | self._right_index -= 1 94 | 95 | self._left_index -= 1 96 | self._fix_indices() 97 | self._arr[self._left_index] = value 98 | 99 | def pop(self): 100 | if len(self) == 0: 101 | raise IndexError("pop from an empty RingBuffer") 102 | self._right_index -= 1 103 | self._fix_indices() 104 | res = self._arr[self._right_index % self._capacity] 105 | return res 106 | 107 | def popleft(self): 108 | if len(self) == 0: 109 | raise IndexError("pop from an empty RingBuffer") 110 | res = self._arr[self._left_index] 111 | self._left_index += 1 112 | self._fix_indices() 113 | return res 114 | 115 | def extend(self, values): 116 | lv = len(values) 117 | if len(self) + lv > self._capacity: 118 | if not self._allow_overwrite: 119 | raise IndexError('extend a RingBuffer such that it would overflow, with overwrite disabled') 120 | elif not len(self): 121 | return 122 | if lv >= self._capacity: 123 | # wipe the entire array! - this may not be threadsafe 124 | self._arr[...] = values[-self._capacity:] 125 | self._right_index = self._capacity 126 | self._left_index = 0 127 | return 128 | 129 | ri = self._right_index % self._capacity 130 | sl1 = np.s_[ri:min(ri + lv, self._capacity)] 131 | sl2 = np.s_[:max(ri + lv - self._capacity, 0)] 132 | self._arr[sl1] = values[:sl1.stop - sl1.start] 133 | self._arr[sl2] = values[sl1.stop - sl1.start:] 134 | self._right_index += lv 135 | 136 | self._left_index = max(self._left_index, self._right_index - self._capacity) 137 | self._fix_indices() 138 | 139 | def extendleft(self, values): 140 | lv = len(values) 141 | if len(self) + lv > self._capacity: 142 | if not self._allow_overwrite: 143 | raise IndexError('extend a RingBuffer such that it would overflow, with overwrite disabled') 144 | elif not len(self): 145 | return 146 | if lv >= self._capacity: 147 | # wipe the entire array! - this may not be threadsafe 148 | self._arr[...] = values[:self._capacity] 149 | self._right_index = self._capacity 150 | self._left_index = 0 151 | return 152 | 153 | self._left_index -= lv 154 | self._fix_indices() 155 | li = self._left_index 156 | sl1 = np.s_[li:min(li + lv, self._capacity)] 157 | sl2 = np.s_[:max(li + lv - self._capacity, 0)] 158 | self._arr[sl1] = values[:sl1.stop - sl1.start] 159 | self._arr[sl2] = values[sl1.stop - sl1.start:] 160 | 161 | self._right_index = min(self._right_index, self._left_index + self._capacity) 162 | 163 | 164 | # implement Sequence methods 165 | def __len__(self): 166 | return self._right_index - self._left_index 167 | 168 | def __getitem__(self, item): 169 | # handle simple (b[1]) and basic (b[np.array([1, 2, 3])]) fancy indexing specially 170 | if not isinstance(item, tuple): 171 | item_arr = np.asarray(item) 172 | if issubclass(item_arr.dtype.type, np.integer): 173 | item_arr = (item_arr + self._left_index) % self._capacity 174 | return self._arr[item_arr] 175 | 176 | # for everything else, get it right at the expense of efficiency 177 | return self._unwrap()[item] 178 | 179 | def __iter__(self): 180 | # alarmingly, this is comparable in speed to using itertools.chain 181 | return iter(self._unwrap()) 182 | 183 | # Everything else 184 | def __repr__(self): 185 | return ''.format(np.asarray(self)) 186 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from io import open 2 | from setuptools import setup 3 | 4 | with open('numpy_ringbuffer/__about__.py', encoding='utf8') as f: 5 | exec(f.read()) 6 | 7 | with open('README.md', encoding='utf8') as f: 8 | long_description = f.read() 9 | 10 | setup( 11 | name="numpy_ringbuffer", 12 | version=__version__, 13 | packages=['numpy_ringbuffer'], 14 | 15 | install_requires=["numpy"], 16 | 17 | author="Eric Wieser", 18 | author_email="wieser.eric+numpy@gmail.com", 19 | description="Ring buffer implementation for numpy", 20 | long_description=long_description, 21 | long_description_content_type='text/markdown', 22 | license="MIT", 23 | keywords=["numpy", "buffer", "ringbuffer", "circular buffer"], 24 | url="https://github.com/eric-wieser/numpy_ringbuffer", 25 | download_url="https://github.com/eric-wieser/numpy_ringbuffer/tarball/v"+__version__, 26 | classifiers=[ 27 | "License :: OSI Approved :: MIT License", 28 | "Intended Audience :: Developers", 29 | "Programming Language :: Python", 30 | "Programming Language :: Python :: 2.7", 31 | "Programming Language :: Python :: 3.5", 32 | "Programming Language :: Python :: 3.6", 33 | "Programming Language :: Python :: 3.7", 34 | "Programming Language :: Python :: 3.8", 35 | "Programming Language :: Python :: 3.9", 36 | "Programming Language :: Python :: 3.10", 37 | ] 38 | ) 39 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | from numpy_ringbuffer import RingBuffer 4 | 5 | class TestAll(unittest.TestCase): 6 | def test_dtype(self): 7 | r = RingBuffer(5) 8 | self.assertEqual(r.dtype, np.dtype(np.float64)) 9 | 10 | r = RingBuffer(5, dtype=np.bool) 11 | self.assertEqual(r.dtype, np.dtype(np.bool)) 12 | 13 | def test_sizes(self): 14 | r = RingBuffer(5, dtype=(int, 2)) 15 | self.assertEqual(r.maxlen, 5) 16 | self.assertEqual(len(r), 0) 17 | self.assertEqual(r.shape, (0,2)) 18 | 19 | r.append([0, 0]) 20 | self.assertEqual(r.maxlen, 5) 21 | self.assertEqual(len(r), 1) 22 | self.assertEqual(r.shape, (1,2)) 23 | 24 | def test_append(self): 25 | r = RingBuffer(5) 26 | 27 | r.append(1) 28 | np.testing.assert_equal(r, np.array([1])) 29 | self.assertEqual(len(r), 1) 30 | 31 | r.append(2) 32 | np.testing.assert_equal(r, np.array([1, 2])) 33 | self.assertEqual(len(r), 2) 34 | 35 | r.append(3) 36 | r.append(4) 37 | r.append(5) 38 | np.testing.assert_equal(r, np.array([1, 2, 3, 4, 5])) 39 | self.assertEqual(len(r), 5) 40 | 41 | r.append(6) 42 | np.testing.assert_equal(r, np.array([2, 3, 4, 5, 6])) 43 | self.assertEqual(len(r), 5) 44 | 45 | self.assertEqual(r[4], 6) 46 | self.assertEqual(r[-1], 6) 47 | 48 | def test_getitem(self): 49 | r = RingBuffer(5) 50 | r.extend([1, 2, 3]) 51 | r.extendleft([4, 5]) 52 | expected = np.array([4, 5, 1, 2, 3]) 53 | np.testing.assert_equal(r, expected) 54 | 55 | for i in range(r.maxlen): 56 | self.assertEqual(expected[i], r[i]) 57 | 58 | ii = [0, 4, 3, 1, 2] 59 | np.testing.assert_equal(r[ii], expected[ii]) 60 | 61 | def test_appendleft(self): 62 | r = RingBuffer(5) 63 | 64 | r.appendleft(1) 65 | np.testing.assert_equal(r, np.array([1])) 66 | self.assertEqual(len(r), 1) 67 | 68 | r.appendleft(2) 69 | np.testing.assert_equal(r, np.array([2, 1])) 70 | self.assertEqual(len(r), 2) 71 | 72 | r.appendleft(3) 73 | r.appendleft(4) 74 | r.appendleft(5) 75 | np.testing.assert_equal(r, np.array([5, 4, 3, 2, 1])) 76 | self.assertEqual(len(r), 5) 77 | 78 | r.appendleft(6) 79 | np.testing.assert_equal(r, np.array([6, 5, 4, 3, 2])) 80 | self.assertEqual(len(r), 5) 81 | 82 | def test_extend(self): 83 | r = RingBuffer(5) 84 | r.extend([1, 2, 3]) 85 | np.testing.assert_equal(r, np.array([1, 2, 3])) 86 | r.popleft() 87 | r.extend([4, 5, 6]) 88 | np.testing.assert_equal(r, np.array([2, 3, 4, 5, 6])) 89 | r.extendleft([0, 1]) 90 | np.testing.assert_equal(r, np.array([0, 1, 2, 3, 4])) 91 | 92 | r.extendleft([1, 2, 3, 4, 5, 6, 7]) 93 | np.testing.assert_equal(r, np.array([1, 2, 3, 4, 5])) 94 | 95 | r.extend([1, 2, 3, 4, 5, 6, 7]) 96 | np.testing.assert_equal(r, np.array([3, 4, 5, 6, 7])) 97 | 98 | def test_pops(self): 99 | r = RingBuffer(3) 100 | r.append(1) 101 | r.appendleft(2) 102 | r.append(3) 103 | np.testing.assert_equal(r, np.array([2, 1, 3])) 104 | 105 | self.assertEqual(r.pop(), 3) 106 | np.testing.assert_equal(r, np.array([2, 1])) 107 | 108 | self.assertEqual(r.popleft(), 2) 109 | np.testing.assert_equal(r, np.array([1])) 110 | 111 | # test empty pops 112 | empty = RingBuffer(1) 113 | with self.assertRaisesRegex(IndexError, "pop from an empty RingBuffer"): 114 | empty.pop() 115 | with self.assertRaisesRegex(IndexError, "pop from an empty RingBuffer"): 116 | empty.popleft() 117 | 118 | def test_2d(self): 119 | r = RingBuffer(5, dtype=(np.float, 2)) 120 | 121 | r.append([1, 2]) 122 | np.testing.assert_equal(r, np.array([[1, 2]])) 123 | self.assertEqual(len(r), 1) 124 | self.assertEqual(np.shape(r), (1, 2)) 125 | 126 | r.append([3, 4]) 127 | np.testing.assert_equal(r, np.array([[1, 2], [3, 4]])) 128 | self.assertEqual(len(r), 2) 129 | self.assertEqual(np.shape(r), (2, 2)) 130 | 131 | r.appendleft([5, 6]) 132 | np.testing.assert_equal(r, np.array([[5, 6], [1, 2], [3, 4]])) 133 | self.assertEqual(len(r), 3) 134 | self.assertEqual(np.shape(r), (3, 2)) 135 | 136 | np.testing.assert_equal(r[0], [5, 6]) 137 | np.testing.assert_equal(r[0,:], [5, 6]) 138 | np.testing.assert_equal(r[:,0], [5, 1, 3]) 139 | 140 | def test_iter(self): 141 | r = RingBuffer(5) 142 | for i in range(5): 143 | r.append(i) 144 | for i, j in zip(r, range(5)): 145 | self.assertEqual(i, j) 146 | 147 | def test_repr(self): 148 | r = RingBuffer(5, dtype=np.int) 149 | for i in range(5): 150 | r.append(i) 151 | 152 | self.assertEqual(repr(r), '') 153 | 154 | def test_no_overwrite(self): 155 | r = RingBuffer(3, allow_overwrite=False) 156 | r.append(1) 157 | r.append(2) 158 | r.appendleft(3) 159 | with self.assertRaisesRegex(IndexError, 'overwrite'): 160 | r.appendleft(4) 161 | with self.assertRaisesRegex(IndexError, 'overwrite'): 162 | r.extendleft([4]) 163 | r.extendleft([]) 164 | 165 | np.testing.assert_equal(r, np.array([3, 1, 2])) 166 | with self.assertRaisesRegex(IndexError, 'overwrite'): 167 | r.append(4) 168 | with self.assertRaisesRegex(IndexError, 'overwrite'): 169 | r.extend([4]) 170 | r.extend([]) 171 | 172 | # works fine if we pop the surplus 173 | r.pop() 174 | r.append(4) 175 | np.testing.assert_equal(r, np.array([3, 1, 4])) 176 | 177 | def test_degenerate(self): 178 | r = RingBuffer(0) 179 | np.testing.assert_equal(r, np.array([])) 180 | 181 | # this does not error with deque(maxlen=0), so should not error here 182 | try: 183 | r.append(0) 184 | r.appendleft(0) 185 | except IndexError: 186 | self.fail() 187 | 188 | if not hasattr(TestAll, 'assertRaisesRegex'): 189 | TestAll.assertRaisesRegex = TestAll.assertRaisesRegexp 190 | 191 | if __name__ == '__main__': 192 | unittest.main() 193 | --------------------------------------------------------------------------------