├── requirements.txt ├── AUTHORS ├── setup.cfg ├── dev-requirements.txt ├── .gitignore ├── .travis.yml ├── pyskiplist ├── __init__.py ├── dllist.py └── skiplist.py ├── docs ├── index.rst ├── Makefile └── conf.py ├── tasks.py ├── tests ├── mem_dllist.py ├── perf_dllist.py ├── documentation.py ├── mem_skiplist.py ├── perf_skiplist.py ├── support.py ├── test_dllist.py └── test_skiplist.py ├── tox.ini ├── LICENSE ├── setup.py ├── runtests.py └── README.rst /requirements.txt: -------------------------------------------------------------------------------- 1 | six 2 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | * Geert Jansen 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | 4 | [flake8] 5 | ignore = E126,E127,E128,E226,E227,E301,E302,E702 6 | max-line-length = 99 7 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | sphinx 3 | wheel 4 | flake8 5 | tox 6 | detox 7 | invoke 8 | coverage 9 | coveralls 10 | twine 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | *.egg-info 4 | *.pyc 5 | *.pyo 6 | .*.swp 7 | .*.swo 8 | *~ 9 | __pycache__ 10 | docs/_build 11 | docs/html 12 | .tox 13 | tests/memory.txt 14 | tests/performance.txt 15 | .coverage 16 | htmlcov 17 | README.html 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3.3 3 | env: 4 | - TOX_ENV=py27 5 | - TOX_ENV=py33 6 | - TOX_ENV=py34 7 | - TOX_ENV=docs 8 | - TOX_ENV=flake8 9 | - TOX_ENV=coverage 10 | install: 11 | - pip install tox 12 | script: 13 | - tox -e $TOX_ENV 14 | - if [ $TOX_ENV = "coverage" ]; then pip install coveralls; coveralls; fi 15 | -------------------------------------------------------------------------------- /pyskiplist/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | # 'from ... import *' is OK in this file. Silence flake8. 9 | # flake8: noqa 10 | 11 | from .skiplist import * 12 | from .dllist import * 13 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | PySkipList Reference Documentation 2 | ================================== 3 | 4 | .. currentmodule:: pyskiplist 5 | 6 | .. autoclass:: pyskiplist.SkipList 7 | :members: 8 | :special-members: 9 | :exclude-members: __init__, __weakref__ 10 | 11 | .. autoclass:: pyskiplist.Node 12 | :members: 13 | 14 | .. autoclass:: pyskiplist.dllist 15 | :members: 16 | :special-members: 17 | :exclude-members: __init__, __weakref__ 18 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | from invoke import run, task 11 | 12 | 13 | @task 14 | def clean(): 15 | run('find . -name __pycache__ | xargs rm -rf || :', echo=True) 16 | run('rm -rf build dist', echo=True) 17 | 18 | 19 | @task(clean) 20 | def develop(): 21 | run('python setup.py develop', echo=True) 22 | -------------------------------------------------------------------------------- /tests/mem_dllist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import unittest 11 | 12 | from pyskiplist import dllist, Node 13 | from pyskiplist.dllist import getsize 14 | from support import MemoryTest 15 | 16 | 17 | class TestDllist(MemoryTest): 18 | 19 | def mem_node(self): 20 | self.add_result(getsize(Node())) 21 | 22 | def mem_dllist(self): 23 | self.add_result(getsize(dllist())) 24 | 25 | 26 | if __name__ == '__main__': 27 | TestDllist.setup_loader() 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /tests/perf_dllist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import time 11 | import unittest 12 | 13 | from pyskiplist import dllist, Node 14 | from support import PerformanceTest 15 | 16 | 17 | class PerfDllist(PerformanceTest): 18 | 19 | def perf_insert_throughput(self): 20 | t0 = t1 = time.time() 21 | count = 0 22 | batch = 1000 23 | dll = dllist() 24 | value = 'foo' 25 | while t1 - t0 < 1: 26 | for i in range(batch): 27 | dll.insert(Node(value)) 28 | count += batch 29 | t1 = time.time() 30 | speed = count / (t1 - t0) 31 | self.add_result(speed) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.defaultTestLoader.testMethodPrefix = 'perf' 36 | unittest.main() 37 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py33, py34, flake8, docs 3 | 4 | [testenv] 5 | commands = python runtests.py {posargs:} unit performance memory 6 | deps = -r{toxinidir}/requirements.txt 7 | # Setting $TOX=yes tells runtests.py to remove the current directory from 8 | # sys.path, so that the tests are guaranteed run against the tox venv. 9 | setenv = TOX=yes 10 | 11 | [testenv:flake8] 12 | commands = flake8 pyskiplist tests setup.py runtests.py tasks.py 13 | deps ={[testenv]deps} 14 | flake8 15 | 16 | [testenv:docs] 17 | commands = python runtests.py {posargs:} documentation 18 | deps = {[testenv]deps} 19 | sphinx 20 | 21 | # Coverage runs in develop mode. This makes sure that the coverage report 22 | # doesn't have long lib/python/site-packages directories inside it. It also 23 | # makes sure that it runs from the tox venv and not from the source dir. 24 | 25 | [testenv:coverage] 26 | usedevelop = True 27 | basepython = python3 28 | commands = 29 | invoke clean 30 | coverage run --source=pyskiplist runtests.py {posargs:} unit 31 | deps = {[testenv]deps} 32 | invoke 33 | coverage 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2015 the PySkipList authors. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /tests/documentation.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import os 11 | import doctest 12 | import unittest 13 | import sphinx 14 | 15 | from support import TestCase 16 | 17 | 18 | class TestDocumentation(TestCase): 19 | 20 | def test_readme(self): 21 | doctest.testfile(os.path.join(self.topdir, 'README.rst'), 22 | module_relative=False, verbose=self.verbose > 2) 23 | 24 | def test_build_docs(self): 25 | docdir = os.path.join(self.topdir, 'docs') 26 | os.chdir(docdir) 27 | htmldir = self.tempdir 28 | args = ['sphinx', '-b', 'html', '-nW', '.', htmldir] 29 | if self.verbose < 3: 30 | args += ['-Q'] 31 | try: 32 | sphinx.main(args) 33 | except SystemExit as e: 34 | ret = e.code 35 | self.assertEqual(ret, 0) 36 | 37 | 38 | if __name__ == '__main__': 39 | os.environ['VERBOSE'] = '3' 40 | unittest.main() 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | from setuptools import setup 11 | 12 | 13 | version_info = { 14 | 'name': 'pyskiplist', 15 | 'version': '1.0.0', 16 | 'description': 'Fast, pure Python indexable skip list', 17 | 'author': 'Geert Jansen', 18 | 'author_email': 'geertj@gmail.com', 19 | 'url': 'https://github.com/geertj/pyskiplist', 20 | 'license': 'MIT', 21 | 'classifiers': [ 22 | 'Development Status :: 5 - Production/Stable', 23 | 'License :: OSI Approved :: MIT License', 24 | 'Operating System :: POSIX', 25 | 'Operating System :: Microsoft :: Windows', 26 | 'Operating System :: MacOS :: MacOS X', 27 | 'Programming Language :: Python :: 2.7', 28 | 'Programming Language :: Python :: 3.3', 29 | 'Programming Language :: Python :: 3.4' 30 | ] 31 | } 32 | 33 | 34 | def main(): 35 | setup(packages=['pyskiplist'], **version_info) 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /tests/mem_skiplist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import sys 11 | import unittest 12 | 13 | from support import MemoryTest 14 | from pyskiplist import SkipList 15 | from pyskiplist.skiplist import getsize 16 | 17 | 18 | class MemSkipList(MemoryTest): 19 | """Memory usage tests for SkipList.""" 20 | 21 | def mem_size(self): 22 | sl = SkipList() 23 | self.add_result(getsize(sl)) 24 | 25 | def mem_node_size(self): 26 | for logN in range(3, 6): 27 | items = 10**logN 28 | sl = SkipList() 29 | for i in range(items): 30 | sl.insert(i, i) 31 | size = getsize(sl) 32 | self.add_result(size/items, suffix=items) 33 | 34 | def mem_node_overhead(self): 35 | for logN in range(3, 6): 36 | items = 10**logN 37 | sl = SkipList() 38 | for i in range(items): 39 | sl.insert(i, i) 40 | overhead = getsize(sl) - items * 2 * sys.getsizeof(i) 41 | self.add_result(overhead/items, suffix=items) 42 | 43 | 44 | if __name__ == '__main__': 45 | MemSkipList.setup_loader() 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import os 11 | import sys 12 | 13 | from argparse import ArgumentParser 14 | from unittest import TestLoader, TextTestRunner, TestSuite 15 | 16 | 17 | parser = ArgumentParser() 18 | parser.add_argument('-v', '--verbose', help='be more verbose', action='count', default=1) 19 | parser.add_argument('-f', '--failfast', help='stop on first failure', action='store_true') 20 | parser.add_argument('-b', '--buffer', help='buffer stdout and stderr', action='store_true') 21 | parser.add_argument('suite', nargs='+', help='name of test suite to run', metavar='suite', 22 | choices=('all', 'unit', 'performance', 'memory', 'documentation')) 23 | args = parser.parse_args() 24 | 25 | if 'all' in args.suite: 26 | args.suite = ['unit', 'performance', 'memory', 'documentation'] 27 | 28 | os.environ['VERBOSE'] = str(args.verbose) 29 | 30 | # Change directory to tests/ irrespective of where we're called from. 31 | topdir = os.path.split(os.path.abspath(__file__))[0] 32 | testdir = os.path.join(topdir, 'tests') 33 | os.chdir(testdir) 34 | 35 | # If running under tox, replace the entry for the current directory on sys.path 36 | # with the test directory. This prevents the tox runs from running in the 37 | # potentially unclean environment from the checkout our source tree. 38 | # Otherwise, if not running under tox, we want the option to run from the 39 | # current directory, so we add the test directory instead. 40 | if os.environ.get('TOX') == 'yes': 41 | sys.path[0] = testdir 42 | else: 43 | sys.path.insert(0, testdir) 44 | 45 | from support import TestCase, MemoryTest, PerformanceTest 46 | 47 | suite = TestSuite() 48 | 49 | for name in args.suite: 50 | TestCase.setup_loader() 51 | if name == 'unit': 52 | pattern = 'test_*.py' 53 | elif name == 'performance': 54 | pattern = 'perf_*.py' 55 | PerformanceTest.setup_loader() 56 | PerformanceTest.start_new_results() 57 | elif name == 'memory': 58 | pattern = 'mem_*.py' 59 | MemoryTest.setup_loader() 60 | MemoryTest.start_new_results() 61 | elif name == 'documentation': 62 | pattern = 'documentation.py' 63 | loader = TestLoader() 64 | tests = loader.discover('.', pattern) 65 | suite.addTest(tests) 66 | 67 | runner = TextTestRunner(verbosity=args.verbose, buffer=args.buffer, failfast=args.failfast) 68 | result = runner.run(suite) 69 | if result.errors or result.failures: 70 | sys.exit(1) 71 | -------------------------------------------------------------------------------- /tests/perf_skiplist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function, division 9 | 10 | import time 11 | import random 12 | import unittest 13 | 14 | from pyskiplist import SkipList 15 | from support import PerformanceTest 16 | 17 | 18 | class PerfSkipList(PerformanceTest): 19 | """Performance tests for our skiplist.""" 20 | 21 | def _create_skiplist(self, n): 22 | # Create a skiplist with *n* elements. 23 | sl = SkipList() 24 | maxkey = 100*n 25 | for i in range(n): 26 | sl.insert(random.randint(0, maxkey), i) 27 | return sl 28 | 29 | def _create_workload(self, sl, n): 30 | # Create a workload with *n* items. 31 | pairs = [] 32 | maxkey = 100*len(sl) 33 | for i in range(n): 34 | pair = (random.randint(0, maxkey), i) 35 | pairs.append(pair) 36 | return pairs 37 | 38 | def perf_search_throughput(self): 39 | for logN in range(3, 6): 40 | items = 10**logN 41 | sl = self._create_skiplist(items) 42 | pairs = list(sl) 43 | random.shuffle(pairs) 44 | load = pairs[0:int(0.2*len(sl))] 45 | count = 0 46 | t0 = t1 = time.time() 47 | while count < len(load) and t1 - t0 < 1: 48 | sl.search(load[count][0]) 49 | count += 1 50 | if count % 100 == 0: 51 | t1 = time.time() 52 | throughput = count / (t1 - t0) 53 | self.add_result(throughput, suffix=items) 54 | 55 | def perf_insert_throughput(self): 56 | for logN in range(3, 6): 57 | items = 10**logN 58 | sl = self._create_skiplist(items) 59 | load = self._create_workload(sl, int(0.2*len(sl))) 60 | count = 0 61 | t0 = t1 = time.time() 62 | while count < len(load) and t1 - t0 < 1: 63 | sl.insert(*load[count]) 64 | count += 1 65 | if count % 100 == 0: 66 | t1 = time.time() 67 | throughput = count / (t1 - t0) 68 | self.add_result(throughput, suffix=items) 69 | 70 | def perf_remove_throughput(self): 71 | for logN in range(3, 6): 72 | items = 10**logN 73 | sl = self._create_skiplist(items) 74 | pairs = list(sl) 75 | random.shuffle(pairs) 76 | load = pairs[0:int(0.2*len(sl))] 77 | count = 0 78 | t0 = t1 = time.time() 79 | while count < len(load) and t1 - t0 < 1: 80 | sl.remove(load[count][0]) 81 | count += 1 82 | if count % 100 == 0: 83 | t1 = time.time() 84 | throughput = count / (t1 - t0) 85 | self.add_result(throughput, suffix=items) 86 | 87 | def perf_index_throughput(self): 88 | for logN in range(3, 6): 89 | items = 10**logN 90 | sl = self._create_skiplist(items) 91 | load = random.sample(range(items), int(0.2*len(sl))) 92 | count = 0 93 | t0 = t1 = time.time() 94 | while count < len(load) and t1 - t0 < 1: 95 | sl[load[count]] 96 | count += 1 97 | if count % 100 == 0: 98 | t1 = time.time() 99 | throughput = count / (t1 - t0) 100 | self.add_result(throughput, suffix=items) 101 | 102 | 103 | if __name__ == '__main__': 104 | PerfSkipList.setup_loader() 105 | unittest.main() 106 | -------------------------------------------------------------------------------- /tests/support.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import os 11 | import sys 12 | import unittest 13 | import tempfile 14 | import shutil 15 | import re 16 | 17 | 18 | __all__ = ['TestCase', 'PerformanceTest', 'MemoryTest'] 19 | 20 | 21 | class TestCase(unittest.TestCase): 22 | """Base class for test suites.""" 23 | 24 | test_prefix = 'test' 25 | 26 | @classmethod 27 | def setup_loader(cls): 28 | unittest.TestLoader.testMethodPrefix = cls.test_prefix 29 | 30 | @classmethod 31 | def setUpClass(cls): 32 | cls.testdir = os.path.abspath(os.path.split(__file__)[0]) 33 | cls.topdir = os.path.split(cls.testdir)[0] 34 | 35 | def setUp(self): 36 | self._tmpindex = 0 37 | self.__tmpdir = None 38 | 39 | def tearDown(self): 40 | if self.__tmpdir is None: 41 | return 42 | # Some paranoia checks to make me feel better before calling 43 | # shutil.rmtree().. 44 | assert '/..' not in self.__tmpdir and '\\..' not in self.__tmpdir 45 | assert os.stat(self.__tmpdir).st_ino == self.__tmpinode 46 | try: 47 | shutil.rmtree(self.__tmpdir) 48 | except OSError: 49 | # On Windows a WindowsError is raised when files are 50 | # still open (WindowsError inherits from OSError). 51 | pass 52 | self.__tmpdir = None 53 | self.__tmpinode = None 54 | 55 | @property 56 | def tempdir(self): 57 | if self.__tmpdir is None: 58 | self.__tmpdir = os.path.realpath(tempfile.mkdtemp('pyskiplist-test')) 59 | self.__tmpinode = os.stat(self.__tmpdir).st_ino 60 | return self.__tmpdir 61 | 62 | def tempname(self, name=None): 63 | if name is None: 64 | name = 'tmpfile-{0}'.format(self._tmpindex) 65 | self._tmpindex += 1 66 | return os.path.join(self.tempdir, name) 67 | 68 | @property 69 | def verbose(self): 70 | try: 71 | return int(os.environ['VERBOSE']) 72 | except (KeyError, ValueError): 73 | return 0 74 | 75 | def assertRaises(self, exc, func, *args, **kwargs): 76 | # Like unittest.assertRaises, but returns the exception. 77 | try: 78 | func(*args, **kwargs) 79 | except exc as e: 80 | exc = e 81 | except Exception as e: 82 | self.fail('Wrong exception raised: {0!s}'.format(e)) 83 | else: 84 | self.fail('Exception not raised: {0!s}'.format(exc)) 85 | return exc 86 | 87 | 88 | re_lu = re.compile('[A-Z]+[a-z0-9]+') 89 | 90 | def split_cap_words(s): 91 | """Split the CamelCase string *s* into words.""" 92 | return re_lu.findall(s) 93 | 94 | 95 | class PerformanceTest(TestCase): 96 | """Base class for performance tests.""" 97 | 98 | results_name = 'performance.txt' 99 | test_prefix = 'perf' 100 | 101 | def add_result(self, result, suffix=None, params={}, name=None): 102 | """Add a performance test result.""" 103 | if name is None: 104 | frame = sys._getframe(1) 105 | clsname = frame.f_locals.get('self', '').__class__.__name__ 106 | methname = frame.f_code.co_name 107 | names = split_cap_words(clsname) 108 | name = '{0}_{1}'.format(''.join(names[1:]), methname[len(self.test_prefix)+1:]).lower() 109 | if suffix is not None: 110 | name = '{}_{!s}'.format(name, suffix) 111 | if params is not None: 112 | params = ','.join(['{0}={1}'.format(k, params[k]) for k in params]) 113 | with open(self.results_name, 'a') as fout: 114 | fout.write('{0:<32s} {1:<16.2f} {2:s}\n'.format(name, result, params)) 115 | 116 | @classmethod 117 | def start_new_results(cls): 118 | try: 119 | os.unlink(cls.results_name) 120 | except OSError: 121 | pass 122 | 123 | 124 | class MemoryTest(PerformanceTest): 125 | """Special case of a performance test that writes to memory.txt.""" 126 | 127 | results_name = 'memory.txt' 128 | test_prefix = 'mem' 129 | -------------------------------------------------------------------------------- /tests/test_dllist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import random 11 | import unittest 12 | import six 13 | 14 | from pyskiplist import dllist, Node 15 | from pyskiplist.dllist import check, dump, getsize 16 | from support import TestCase 17 | 18 | 19 | class TestDllist(TestCase): 20 | 21 | def test_basic(self): 22 | dll = dllist() 23 | check(dll) 24 | self.assertIsNone(dll.first) 25 | self.assertIsNone(dll.last) 26 | self.assertEqual(len(dll), 0) 27 | # insert first element 28 | n1 = Node('foo') 29 | dll.insert(n1) 30 | self.assertIs(dll.first, n1) 31 | self.assertIs(dll.last, n1) 32 | self.assertEqual(len(dll), 1) 33 | check(dll) 34 | # insert second at end 35 | n2 = Node('bar') 36 | dll.insert(n2) 37 | self.assertIs(dll.first, n1) 38 | self.assertIs(dll.last, n2) 39 | self.assertEqual(len(dll), 2) 40 | check(dll) 41 | # insert in middle 42 | n3 = Node('baz') 43 | dll.insert(n3, before=n2) 44 | self.assertIs(dll.first, n1) 45 | self.assertIs(dll.last, n2) 46 | self.assertEqual(len(dll), 3) 47 | check(dll) 48 | # remove middle 49 | dll.remove(n3) 50 | self.assertIs(dll.first, n1) 51 | self.assertIs(dll.last, n2) 52 | self.assertEqual(len(dll), 2) 53 | check(dll) 54 | # remove first 55 | dll.remove(n1) 56 | self.assertIs(dll.first, n2) 57 | self.assertIs(dll.last, n2) 58 | self.assertEqual(len(dll), 1) 59 | check(dll) 60 | # remove remaining element 61 | dll.remove(n2) 62 | self.assertIsNone(dll.first) 63 | self.assertIsNone(dll.last) 64 | self.assertEqual(len(dll), 0) 65 | check(dll) 66 | 67 | def test_remove_removed(self): 68 | # It is OK to remove an already removed node. 69 | dll = dllist() 70 | node = Node('foo') 71 | dll.insert(node) 72 | self.assertIn(node, list(dll)) 73 | dll.remove(node) 74 | self.assertNotIn(node, list(dll)) 75 | dll.remove(node) 76 | check(dll) 77 | 78 | def test_iter(self): 79 | dll = dllist() 80 | for i in range(10): 81 | dll.insert(Node(10+i)) 82 | check(dll) 83 | value = 10 84 | for node in dll: 85 | self.assertIsInstance(node, Node) 86 | self.assertEqual(node.value, value) 87 | value += 1 88 | check(dll) 89 | 90 | def test_many_nodes(self): 91 | nodes = [] 92 | dll = dllist() 93 | count = 10000 94 | for i in range(count): 95 | before = random.choice(nodes) if nodes else None 96 | node = Node(i) 97 | dll.insert(node, before) 98 | nodes.append(node) 99 | self.assertEqual(len(dll), i+1) 100 | if i % 100 == 0: 101 | check(dll) 102 | check(dll) 103 | for i in range(count): 104 | r = random.randint(0, len(nodes)-1) 105 | node = nodes[r]; del nodes[r] 106 | dll.remove(node) 107 | self.assertEqual(len(dll), count-i-1) 108 | if i % 100 == 0: 109 | check(dll) 110 | check(dll) 111 | 112 | def test_size(self): 113 | dll = dllist() 114 | size = getsize(dll) 115 | self.assertIsInstance(size, int) 116 | self.assertGreater(size, 0) 117 | self.assertLess(size, 200) 118 | 119 | def test_node_size(self): 120 | node = Node('foo') 121 | size = getsize(node) 122 | self.assertIsInstance(size, int) 123 | self.assertGreater(size, 0) 124 | self.assertLess(size, 200) 125 | 126 | 127 | class TestDllistDebug(TestCase): 128 | """Coverage for debugging tools.""" 129 | 130 | def test_dump(self): 131 | dll = dllist() 132 | dll.insert(Node('foo')) 133 | dll.insert(Node('bar')) 134 | out = six.StringIO() 135 | dump(dll, out) 136 | s = out.getvalue() 137 | self.assertIsInstance(s, str) 138 | self.assertGreater(len(s), 20) 139 | 140 | 141 | if __name__ == '__main__': 142 | unittest.main() 143 | -------------------------------------------------------------------------------- /pyskiplist/dllist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import sys 11 | 12 | __all__ = ['Node', 'dllist'] 13 | 14 | 15 | if __debug__: 16 | 17 | def dump(dll, file=sys.stdout): 18 | print('== Dumping dllist {!r}'.format(dll), file=file) 19 | print('Size: {}'.format(dll._size), file=file) 20 | print('First: {!r}'.format(dll.first if dll.first else None), file=file) 21 | print('Last: {!r}'.format(dll.last if dll.last else None), file=file) 22 | print('Nodes:', file=file) 23 | count = 0 24 | node = dll.first 25 | while node is not None: 26 | print('- {!r} [{}]'.format(node, count), file=file) 27 | node = node._next 28 | count += 1 29 | print('Total nodes: {}'.format(count), file=file) 30 | 31 | def check(dll): 32 | if dll.first is None: 33 | assert dll.last is None 34 | assert dll._size == 0 35 | return 36 | node = dll.first 37 | assert node._prev is None 38 | nnode = node._next 39 | count = 1 40 | while nnode is not None: 41 | assert nnode._prev is node 42 | node, nnode = nnode, nnode._next 43 | count += 1 44 | assert node is dll.last 45 | assert count == dll._size 46 | 47 | def getsize(obj): 48 | """Return the size of a Node or dllist.""" 49 | size = sys.getsizeof(obj) 50 | for key in obj.__slots__: 51 | size += sys.getsizeof(getattr(obj, key)) 52 | return size 53 | 54 | 55 | class Node(object): 56 | """Base node class for :class:`dllist`. 57 | 58 | You can create a custom node with extra attributes by inheriting from this 59 | class. When you do this you need to set the ``'__slots__'`` attribute to 60 | include your custom attributes, and include ``'_prev'`` and ``'_next'`` also. 61 | """ 62 | 63 | __slots__ = ('_prev', '_next', 'value') 64 | 65 | def __repr__(self): 66 | return '' \ 67 | .format(id(self._prev), id(self._next), self.value) 68 | 69 | def __init__(self, value=None): 70 | self._prev = None 71 | self._next = None 72 | self.value = value 73 | 74 | 75 | class dllist(object): 76 | """A doubly linked list.""" 77 | 78 | __slots__ = ('_first', '_last', '_size') 79 | 80 | def __init__(self): 81 | self._first = None 82 | self._last = None 83 | self._size = 0 84 | 85 | @property 86 | def first(self): 87 | """The first node in the list.""" 88 | return self._first 89 | 90 | @property 91 | def last(self): 92 | """The last node in the list.""" 93 | return self._last 94 | 95 | def __len__(self): 96 | """Return the number of nodes in this list.""" 97 | return self._size 98 | 99 | def remove(self, node): 100 | """Remove a node from the list. 101 | 102 | The *node* argument must be a node that was previously inserted in the 103 | list 104 | """ 105 | if node is None or node._prev == -1: 106 | return 107 | if node._next is None: 108 | self._last = node._prev # last node 109 | else: 110 | node._next._prev = node._prev 111 | if node._prev is None: 112 | self._first = node._next # first node 113 | else: 114 | node._prev._next = node._next 115 | node._prev = node._next = -1 116 | self._size -= 1 117 | 118 | def insert(self, node, before=None): 119 | """Insert a new node in the list. 120 | 121 | The *node* argument must be a :class:`Node` instance. 122 | 123 | If *before* is not provided (the default), the node is appended at the 124 | end of the list. If *before* is provided, it must be a :class:`Node` 125 | instance that is already part of this list, and the node is inserted 126 | before this node. 127 | 128 | To insert at the start of the list, set *before* to :attr:`first`. 129 | """ 130 | if self._first is None: 131 | self._first = self._last = node # first node in list 132 | self._size += 1 133 | return node 134 | if before is None: 135 | self._last._next = node # insert as last node 136 | node._prev = self._last 137 | self._last = node 138 | else: 139 | node._next = before 140 | node._prev = before._prev 141 | if node._prev: 142 | node._prev._next = node 143 | else: 144 | self._first = node # inserting as first node 145 | node._next._prev = node 146 | self._size += 1 147 | 148 | def __iter__(self): 149 | """Return an iterator/generator that yields all nodes. 150 | 151 | Note: it is safe to remove the current node while iterating but you 152 | should not remove the next one. 153 | """ 154 | node = self._first 155 | while node is not None: 156 | next_node = node._next 157 | yield node 158 | node = next_node 159 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Welcome to PySkipList 2 | ===================== 3 | 4 | .. image:: https://travis-ci.org/geertj/pyskiplist.svg?branch=master 5 | :target: https://travis-ci.org/geertj/pyskiplist 6 | 7 | .. image:: https://coveralls.io/repos/geertj/pyskiplist/badge.svg?branch=master 8 | :target: https://coveralls.io/r/geertj/pyskiplist 9 | 10 | .. image:: https://readthedocs.org/projects/pyskiplist/badge/?version=stable 11 | :target: https://readthedocs.org/projects/pyskiplist/?badge=stable 12 | 13 | .. image:: https://badge.fury.io/py/pyskiplist.svg 14 | :target: http://badge.fury.io/py/pyskiplist 15 | 16 | Note (July 2015) 17 | ---------------- 18 | 19 | On modern architectures, memory can be copied or moved very fast if done 20 | sequentially. This is mostly due to the availability of very wide SIMD 21 | instructions and byte cache line of 64 bytes or more. Because of this, 22 | algorithms that operate on data in a more sequential fashion may be more 23 | efficient than algorithms that don't, even if the sequential access means that 24 | sometimes larger chunks of data have to be copied or moved. 25 | 26 | B-tree data structures were originally specifically created for rotating 27 | storage that has slow seek times but high throughput. But because of the memory 28 | characteristics mentioned above, B-tree structures can actually be more 29 | efficient that binary trees (RB-trees, AVL trees, 2-3 trees) and skip list for 30 | in-memory applications. 31 | 32 | An example of such an in-memory B-tree implementation that serves the same use 33 | case as PySkipList is SortedContainers_. In addition to being faster than 34 | PySkipList, it also has the benefit that it uses less memory allocations and 35 | less memory overall, and that it doesn't need C code for acceleration. For some 36 | discussion on performance and memory usage, see #1. 37 | 38 | The PySkipList code is still a good example of how you can build an efficient, 39 | indexable skip list in pure Python code. 40 | 41 | Original README is below: 42 | 43 | Overview 44 | -------- 45 | 46 | PySkipList is a fast, pure Python implementation of an indexable skiplist. It 47 | implements a ``SkipList`` data structure that provides an always sorted, 48 | list-like data structure for (key, value) pairs. It efficiently supports the 49 | following operations: 50 | 51 | * Insert a pair in the list, maintaining sorted order. 52 | * Find the value of a given key. 53 | * Remove a given pair based on a key. 54 | * Iterate over all pairs in sorted order. 55 | * Find the position of a given key. 56 | * Access a pair at a certain position. 57 | * Delete a pair at a certain position. 58 | 59 | Since PySkipList is a pure Python implementation, it should work well on 60 | alternative Python implementations such as PyPy and Jython. 61 | 62 | 63 | Example 64 | ------- 65 | 66 | The following provides a few examples on how to use the ``SkipList`` API:: 67 | 68 | >>> from pyskiplist import SkipList 69 | >>> sl = SkipList() 70 | >>> sl.insert('foo', 'bar') 71 | >>> sl.insert('baz', 'qux') 72 | >>> sl 73 | SkipList((('baz', 'qux'), ('foo', 'bar'))) 74 | >>> sl.search('foo') 75 | 'bar' 76 | >>> sl[0] 77 | ('baz', 'qux') 78 | >>> sl.remove('foo') # remove by key 79 | >>> del sl[0] # remove by position 80 | 81 | 82 | Full documentation can be found on http://pyskiplist.readthedocs.org/. 83 | 84 | Asymptotic Complexity 85 | --------------------- 86 | 87 | Below are the Big-O complexities of the various operations implemented by 88 | pyskiplist: 89 | 90 | ================== ========== 91 | Operation Complexity 92 | ================== ========== 93 | insertion O(log N) 94 | search by key O(log N) 95 | removal by key O(log N) 96 | forward iteration O(1) 97 | find by position O(log N) 98 | access by position O(log N) 99 | delete by position O(log N) 100 | ================== ========== 101 | 102 | 103 | Performance 104 | ----------- 105 | 106 | Below are the results of some performance tests. These are for Python 3.4.2 on 107 | my Linux laptop: 108 | 109 | =================== =================== 110 | Test Operations / second 111 | =================== =================== 112 | Insert @ 1k nodes 45,056 113 | Insert @ 10k nodes 42,137 114 | Insert @ 100k nodes 28,086 115 | Remove @ 1k nodes 54,316 116 | Remove @ 10k nodes 46,240 117 | Remove @ 100k nodes 35,114 118 | Search @ 1k nodes 137,248 119 | Search @ 10k nodes 109,480 120 | Search @ 100k nodes 77,939 121 | =================== =================== 122 | 123 | 124 | Memory usage 125 | ------------ 126 | 127 | PySkipList tries to be efficient with regards to memory usage. The numbers 128 | below are for Python 3.4.2 on my Linux laptop. This specific test stores pairs 129 | of integer keys and an integer values in a skiplist. The total size of the two 130 | integers on this Python version is 56 bytes. 131 | 132 | ===== ============ ================= 133 | Nodes Bytes / node Overhead (fixed) 134 | ===== ============ ================= 135 | 1k 164 108 136 | 10k 162 106 137 | 100k 162 106 138 | ===== ============ ================= 139 | 140 | 141 | Implementation notes 142 | -------------------- 143 | 144 | Reference papers on skiplists: 145 | 146 | * ftp://ftp.cs.umd.edu/pub/skipLists/skiplists.pdf (original paper) 147 | * http://drum.lib.umd.edu/bitstream/1903/544/2/CS-TR-2286.1.pdf (cookbook) 148 | 149 | This implementation uses a novel (as far as I know) technique where it stores 150 | just a single link width per node, and only in nodes with level > 1. The link 151 | corresponds to the number of nodes skipped by the highest incoming link. Other 152 | implementations that I've seen all store a width for every link. This approach 153 | saves a lot of memory. The overhead should just be 1/e (0.37) integers per 154 | node. It makes an indexable skiplist almost as memory efficient as its 155 | non-indexable cousin. 156 | 157 | Duplicate keys are allowed in this implementation, and insertion order is 158 | maintained. 159 | 160 | Skiplist nodes are implemented as plain lists instead of objects. This saves 161 | memory. Kudos to http://pythonsweetness.tumblr.com/post/45227295342 for the 162 | idea. 163 | 164 | The built-in Mersenne Twister is used as the random source. This is preferable 165 | over SystemRandom since it doesn't require a system call and there is no need 166 | for cryptographically secure numbers. 167 | 168 | 169 | .. _SortedContainers: https://pypi.python.org/pypi/sortedcontainers 170 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PySkipList.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PySkipList.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/PySkipList" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PySkipList" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # PySkipList documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jun 23 09:30:51 2015. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | import shlex 19 | 20 | # Get the version from the seutp.py file. 21 | sys.path.insert(0, os.path.abspath('..')) 22 | from setup import version_info 23 | 24 | # If extensions (or modules to document with autodoc) are in another directory, 25 | # add these directories to sys.path here. If the directory is relative to the 26 | # documentation root, use os.path.abspath to make it absolute, like shown here. 27 | #sys.path.insert(0, os.path.abspath('.')) 28 | 29 | # -- General configuration ------------------------------------------------ 30 | 31 | # If your documentation needs a minimal Sphinx version, state it here. 32 | #needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.autodoc', 39 | ] 40 | 41 | autoclass_content = 'both' 42 | autodoc_member_order = 'bysource' 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ['_templates'] 46 | 47 | # The suffix(es) of source filenames. 48 | # You can specify multiple suffix as a list of string: 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = '.rst' 51 | 52 | # The encoding of source files. 53 | #source_encoding = 'utf-8-sig' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # General information about the project. 59 | project = 'PySkipList' 60 | copyright = '2012-2015, Geert Jansen' 61 | author = 'Geert Jansen' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = version_info['version'] 69 | # The full version, including alpha/beta/rc tags. 70 | release = version 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = None 78 | 79 | # There are two options for replacing |today|: either, you set today to some 80 | # non-false value, then it is used: 81 | #today = '' 82 | # Else, today_fmt is used as the format for a strftime call. 83 | #today_fmt = '%B %d, %Y' 84 | 85 | # List of patterns, relative to source directory, that match files and 86 | # directories to ignore when looking for source files. 87 | exclude_patterns = ['_build'] 88 | 89 | # The reST default role (used for this markup: `text`) to use for all 90 | # documents. 91 | #default_role = None 92 | 93 | # If true, '()' will be appended to :func: etc. cross-reference text. 94 | #add_function_parentheses = True 95 | 96 | # If true, the current module name will be prepended to all description 97 | # unit titles (such as .. function::). 98 | #add_module_names = True 99 | 100 | # If true, sectionauthor and moduleauthor directives will be shown in the 101 | # output. They are ignored by default. 102 | #show_authors = False 103 | 104 | # The name of the Pygments (syntax highlighting) style to use. 105 | pygments_style = 'sphinx' 106 | 107 | # A list of ignored prefixes for module index sorting. 108 | #modindex_common_prefix = [] 109 | 110 | # If true, keep warnings as "system message" paragraphs in the built documents. 111 | #keep_warnings = False 112 | 113 | # If true, `todo` and `todoList` produce output, else they produce nothing. 114 | todo_include_todos = False 115 | 116 | 117 | # -- Options for HTML output ---------------------------------------------- 118 | 119 | # The theme to use for HTML and HTML Help pages. See the documentation for 120 | # a list of builtin themes. 121 | html_theme = 'pyramid' 122 | 123 | # Theme options are theme-specific and customize the look and feel of a theme 124 | # further. For a list of options available for each theme, see the 125 | # documentation. 126 | #html_theme_options = {} 127 | 128 | # Add any paths that contain custom themes here, relative to this directory. 129 | #html_theme_path = [] 130 | 131 | # The name for this set of Sphinx documents. If None, it defaults to 132 | # " v documentation". 133 | #html_title = None 134 | 135 | # A shorter title for the navigation bar. Default is the same as html_title. 136 | #html_short_title = None 137 | 138 | # The name of an image file (relative to this directory) to place at the top 139 | # of the sidebar. 140 | #html_logo = None 141 | 142 | # The name of an image file (within the static path) to use as favicon of the 143 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 144 | # pixels large. 145 | #html_favicon = None 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = [] 151 | 152 | # Add any extra paths that contain custom files (such as robots.txt or 153 | # .htaccess) here, relative to this directory. These files are copied 154 | # directly to the root of the documentation. 155 | #html_extra_path = [] 156 | 157 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 158 | # using the given strftime format. 159 | #html_last_updated_fmt = '%b %d, %Y' 160 | 161 | # If true, SmartyPants will be used to convert quotes and dashes to 162 | # typographically correct entities. 163 | #html_use_smartypants = True 164 | 165 | # Custom sidebar templates, maps document names to template names. 166 | #html_sidebars = {} 167 | 168 | # Additional templates that should be rendered to pages, maps page names to 169 | # template names. 170 | #html_additional_pages = {} 171 | 172 | # If false, no module index is generated. 173 | #html_domain_indices = True 174 | 175 | # If false, no index is generated. 176 | #html_use_index = True 177 | 178 | # If true, the index is split into individual pages for each letter. 179 | #html_split_index = False 180 | 181 | # If true, links to the reST sources are added to the pages. 182 | #html_show_sourcelink = True 183 | 184 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 185 | #html_show_sphinx = True 186 | 187 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 188 | #html_show_copyright = True 189 | 190 | # If true, an OpenSearch description file will be output, and all pages will 191 | # contain a tag referring to it. The value of this option must be the 192 | # base URL from which the finished HTML is served. 193 | #html_use_opensearch = '' 194 | 195 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 196 | #html_file_suffix = None 197 | 198 | # Language to be used for generating the HTML full-text search index. 199 | # Sphinx supports the following languages: 200 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 201 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' 202 | #html_search_language = 'en' 203 | 204 | # A dictionary with options for the search language support, empty by default. 205 | # Now only 'ja' uses this config value 206 | #html_search_options = {'type': 'default'} 207 | 208 | # The name of a javascript file (relative to the configuration directory) that 209 | # implements a search results scorer. If empty, the default will be used. 210 | #html_search_scorer = 'scorer.js' 211 | 212 | # Output file base name for HTML help builder. 213 | htmlhelp_basename = 'PySkipListdoc' 214 | 215 | # -- Options for LaTeX output --------------------------------------------- 216 | 217 | latex_elements = { 218 | # The paper size ('letterpaper' or 'a4paper'). 219 | #'papersize': 'letterpaper', 220 | 221 | # The font size ('10pt', '11pt' or '12pt'). 222 | #'pointsize': '10pt', 223 | 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | 227 | # Latex figure (float) alignment 228 | #'figure_align': 'htbp', 229 | } 230 | 231 | # Grouping the document tree into LaTeX files. List of tuples 232 | # (source start file, target name, title, 233 | # author, documentclass [howto, manual, or own class]). 234 | latex_documents = [ 235 | (master_doc, 'PySkipList.tex', 'PySkipList Documentation', 236 | 'Geert Jansen', 'manual'), 237 | ] 238 | 239 | # The name of an image file (relative to this directory) to place at the top of 240 | # the title page. 241 | #latex_logo = None 242 | 243 | # For "manual" documents, if this is true, then toplevel headings are parts, 244 | # not chapters. 245 | #latex_use_parts = False 246 | 247 | # If true, show page references after internal links. 248 | #latex_show_pagerefs = False 249 | 250 | # If true, show URL addresses after external links. 251 | #latex_show_urls = False 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #latex_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #latex_domain_indices = True 258 | 259 | 260 | # -- Options for manual page output --------------------------------------- 261 | 262 | # One entry per manual page. List of tuples 263 | # (source start file, name, description, authors, manual section). 264 | man_pages = [ 265 | (master_doc, 'pyskiplist', 'PySkipList Documentation', 266 | [author], 1) 267 | ] 268 | 269 | # If true, show URL addresses after external links. 270 | #man_show_urls = False 271 | 272 | 273 | # -- Options for Texinfo output ------------------------------------------- 274 | 275 | # Grouping the document tree into Texinfo files. List of tuples 276 | # (source start file, target name, title, author, 277 | # dir menu entry, description, category) 278 | texinfo_documents = [ 279 | (master_doc, 'PySkipList', 'PySkipList Documentation', 280 | author, 'PySkipList', 'One line description of project.', 281 | 'Miscellaneous'), 282 | ] 283 | 284 | # Documents to append as an appendix to all manuals. 285 | #texinfo_appendices = [] 286 | 287 | # If false, no module index is generated. 288 | #texinfo_domain_indices = True 289 | 290 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 291 | #texinfo_show_urls = 'footnote' 292 | 293 | # If true, do not generate a @detailmenu in the "Top" node's menu. 294 | #texinfo_no_detailmenu = False 295 | -------------------------------------------------------------------------------- /tests/test_skiplist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of PySkiplist. PySkiplist is Copyright (c) 2012-2015 by 3 | # the PySkiplist authors. 4 | # 5 | # PySkiplist is free software available under the MIT license. See the file 6 | # named LICENSE distributed with this file for the exact licensing terms. 7 | 8 | from __future__ import absolute_import, print_function 9 | 10 | import random 11 | import unittest 12 | import six 13 | 14 | from support import TestCase 15 | from pyskiplist import SkipList 16 | from pyskiplist.skiplist import check, dump, getsize 17 | 18 | 19 | class TestSkipList(TestCase): 20 | """Unit test suite for SkipList.""" 21 | 22 | size = 100 23 | 24 | def _create_skiplist(self, size, keysize, valuesize): 25 | sl = SkipList() 26 | pairs = [] 27 | values = {} 28 | for i in range(size): 29 | pair = (random.randint(0, keysize), random.randint(0, valuesize)) 30 | sl.insert(*pair) 31 | pairs.append(pair) 32 | if pair[0] not in values: 33 | values[pair[0]] = [] 34 | values[pair[0]].append(pair[1]) 35 | pairs = sorted(pairs, key=lambda x: x[0]) 36 | return sl, pairs, values 37 | 38 | # GENERAL API ... 39 | 40 | def test_level(self): 41 | sl = SkipList() 42 | self.assertEqual(sl.level, 1) 43 | check(sl) 44 | 45 | def test_insert(self): 46 | size = self.size 47 | sl = SkipList() 48 | pairs = [] 49 | for i in range(size): 50 | pair = (random.randint(0, 2*size), random.randint(0, 10*size)) 51 | sl.insert(*pair) 52 | pairs = sorted(pairs + [pair], key=lambda x: x[0]) 53 | check(sl); self.assertEqual(list(sl), pairs) 54 | self.assertGreater(sl.level, 1) 55 | 56 | def test_replace(self): 57 | size = self.size 58 | sl = SkipList() 59 | values = {} 60 | for i in range(size): 61 | pair = (random.randint(0, 2*size), random.randint(0, 10*size)) 62 | sl.replace(*pair) 63 | values[pair[0]] = pair[1] 64 | pairs = sorted(values.items(), key=lambda x: x[0]) 65 | check(sl); self.assertEqual(list(sl), pairs) 66 | self.assertGreater(sl.level, 1) 67 | 68 | def test_clear(self): 69 | size = self.size 70 | sl = SkipList() 71 | for i in range(size): 72 | sl.insert(random.randint(0, 2*size), random.randint(0, 10*size)) 73 | self.assertGreater(sl.level, 1) 74 | self.assertEqual(len(sl), size) 75 | sl.clear() 76 | check(sl); self.assertEqual(list(sl), []) 77 | self.assertEqual(sl.level, 1) 78 | 79 | def test_len(self): 80 | size = self.size 81 | sl = SkipList() 82 | pairs = [] 83 | for i in range(size): 84 | pair = (random.randint(0, 2*size), random.randint(0, 10*size)) 85 | sl.insert(*pair) 86 | pairs = sorted(pairs + [pair], key=lambda x: x[0]) 87 | self.assertEqual(len(sl), i+1) 88 | check(sl); self.assertEqual(list(sl), pairs) 89 | 90 | def test_bool(self): 91 | sl = SkipList() 92 | self.assertFalse(sl) 93 | self.assertFalse(bool(sl)) 94 | check(sl) 95 | sl.insert('foo', 'bar') 96 | self.assertTrue(sl) 97 | self.assertTrue(bool(sl)) 98 | check(sl) 99 | 100 | def test_repr(self): 101 | sl = SkipList() 102 | sl.insert(1, 2) 103 | sl.insert(3, 4) 104 | self.assertEqual(repr(sl), 'SkipList(((1, 2), (3, 4)))') 105 | check(sl) 106 | 107 | def test_iter(self): 108 | size = self.size 109 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 110 | self.assertEqual(list(sl), pairs) 111 | check(sl) 112 | 113 | def test_items(self): 114 | size = self.size 115 | sl, pairs, values = self._create_skiplist(size, size, 10*size) 116 | # test .items(), .keys() and .values() 117 | for ix, func in ((slice(0, 2), sl.items), (0, sl.keys), (1, sl.values)): 118 | def ref(start, stop): 119 | return [pair[ix] for pair in pairs 120 | if (start is None or pair[0] >= start) 121 | and (stop is None or pair[0] < stop)] 122 | self.assertEqual(list(func()), ref(None, None)) 123 | self.assertEqual(list(func(start=10)), ref(10, None)) 124 | self.assertEqual(list(func(start=10.1)), ref(10.1, None)) 125 | self.assertEqual(list(func(start=11)), ref(11, None)) 126 | self.assertEqual(list(func(stop=90)), ref(None, 90)) 127 | self.assertEqual(list(func(stop=90.1)), ref(None, 90.1)) 128 | self.assertEqual(list(func(stop=91)), ref(None, 91)) 129 | self.assertEqual(list(func(start=10, stop=90)), ref(10, 90)) 130 | self.assertEqual(list(func(start=10.1, stop=90)), ref(10.1, 90)) 131 | self.assertEqual(list(func(start=10, stop=90.1)), ref(10, 90.1)) 132 | self.assertEqual(list(func(start=10.1, stop=90.1)), ref(10.1, 90.1)) 133 | check(sl); self.assertEqual(list(sl), pairs) 134 | 135 | def test_popitem(self): 136 | size = self.size 137 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 138 | while pairs: 139 | self.assertEqual(sl.popitem(), pairs[0]) 140 | del pairs[0] 141 | check(sl); self.assertEqual(list(sl), pairs) 142 | self.assertRaises(KeyError, sl.popitem) 143 | check(sl); self.assertEqual(list(sl), pairs) 144 | 145 | # KEY BASED API ... 146 | 147 | def test_search(self): 148 | size = self.size 149 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 150 | for key in values: 151 | self.assertEqual(sl.search(key), values[key][0]) 152 | check(sl); self.assertEqual(list(sl), pairs) 153 | self.assertIsNone(sl.search(random.randint(3*size, 10*size))) 154 | check(sl); self.assertEqual(list(sl), pairs) 155 | self.assertEqual(sl.search(random.randint(3*size, 10*size), -1), -1) 156 | check(sl); self.assertEqual(list(sl), pairs) 157 | 158 | def test_remove(self): 159 | size = self.size 160 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 161 | for key in values: 162 | for value in values[key]: 163 | sl.remove(key) 164 | pairs.remove((key, value)) 165 | check(sl); self.assertEqual(list(sl), pairs) 166 | self.assertRaises(KeyError, sl.remove, key) 167 | check(sl); self.assertEqual(list(sl), pairs) 168 | 169 | def test_pop(self): 170 | size = self.size 171 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 172 | for key in values: 173 | for value in values[key]: 174 | self.assertEqual(sl.pop(key), value) 175 | pairs.remove((key, value)) 176 | check(sl); self.assertEqual(list(sl), pairs) 177 | self.assertRaises(KeyError, sl.pop, key) 178 | check(sl); self.assertEqual(list(sl), pairs) 179 | self.assertIsNone(sl.pop(key, None)) 180 | check(sl); self.assertEqual(list(sl), pairs) 181 | self.assertEqual(sl.pop(key, -1), -1) 182 | check(sl); self.assertEqual(list(sl), pairs) 183 | 184 | def test_contains(self): 185 | size = self.size 186 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 187 | for key in values: 188 | self.assertIn(key, sl) 189 | check(sl); self.assertEqual(list(sl), pairs) 190 | self.assertNotIn(random.randint(3*size, 10*size), sl) 191 | check(sl); self.assertEqual(list(sl), pairs) 192 | 193 | def test_index(self): 194 | size = self.size 195 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 196 | for key in values: 197 | self.assertEqual(sl.index(key), pairs.index((key, values[key][0]))) 198 | check(sl); self.assertEqual(list(sl), pairs) 199 | self.assertRaises(KeyError, sl.index, random.randint(3*size, 10*size)) 200 | check(sl); self.assertEqual(list(sl), pairs) 201 | 202 | def test_count(self): 203 | size = self.size 204 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 205 | for key in values: 206 | self.assertEqual(sl.count(key), len(values[key])) 207 | check(sl); self.assertEqual(list(sl), pairs) 208 | self.assertEqual(sl.count(random.randint(3*size, 10*size)), 0) 209 | check(sl); self.assertEqual(list(sl), pairs) 210 | 211 | # BY POSITION API ... 212 | 213 | def test_getitem(self): 214 | size = self.size 215 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 216 | for i in range(size): 217 | self.assertEqual(sl[i], pairs[i]) 218 | check(sl); self.assertEqual(list(sl), pairs) 219 | self.assertEqual(sl[-i-1], pairs[-i-1]) 220 | check(sl); self.assertEqual(list(sl), pairs) 221 | self.assertRaises(IndexError, sl.__getitem__, size) 222 | check(sl); self.assertEqual(list(sl), pairs) 223 | self.assertRaises(IndexError, sl.__getitem__, -size-1) 224 | check(sl); self.assertEqual(list(sl), pairs) 225 | self.assertRaises(TypeError, sl.__getitem__, 'foo') 226 | check(sl); self.assertEqual(list(sl), pairs) 227 | 228 | def test_getitem_slice(self): 229 | size = self.size 230 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 231 | for ix in (slice(None, None), slice(None, 10), slice(10, None), 232 | slice(10, 90), slice(10, -10), slice(-10, None), 233 | slice(-10, -1), slice(None, -10), slice(None, -1)): 234 | self.assertEqual(list(sl[ix]), pairs[ix]) 235 | check(sl); self.assertEqual(list(sl), pairs) 236 | 237 | def test_delitem(self): 238 | size = self.size 239 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 240 | while pairs: 241 | ix = random.randrange(-len(pairs), len(pairs)) 242 | del sl[ix] 243 | del pairs[ix] 244 | check(sl); self.assertEqual(list(sl), pairs) 245 | self.assertRaises(IndexError, sl.__delitem__, len(pairs)) 246 | check(sl); self.assertEqual(list(sl), pairs) 247 | self.assertRaises(IndexError, sl.__delitem__, -len(pairs)-1) 248 | check(sl); self.assertEqual(list(sl), pairs) 249 | self.assertRaises(TypeError, sl.__delitem__, 'foo') 250 | check(sl); self.assertEqual(list(sl), pairs) 251 | 252 | def test_setitem(self): 253 | size = self.size 254 | sl, pairs, values = self._create_skiplist(size, 2*size, 10*size) 255 | for ix, pair in enumerate(pairs): 256 | sl[ix] = 2*pair[1] 257 | pairs[ix] = (pair[0], 2*pair[1]) 258 | check(sl); self.assertEqual(list(sl), pairs) 259 | self.assertRaises(IndexError, sl.__setitem__, size, None) 260 | check(sl); self.assertEqual(list(sl), pairs) 261 | self.assertRaises(IndexError, sl.__setitem__, -size-1, None) 262 | check(sl); self.assertEqual(list(sl), pairs) 263 | self.assertRaises(TypeError, sl.__setitem__, 'foo', None) 264 | check(sl); self.assertEqual(list(sl), pairs) 265 | 266 | 267 | class TestSkipListDebug(TestCase): 268 | """Coverage for debugging tools.""" 269 | 270 | def test_size(self): 271 | sl = SkipList() 272 | sl.insert('foo', 'bar') 273 | size = getsize(sl) 274 | self.assertIsInstance(size, int) 275 | self.assertGreater(size, 0) 276 | self.assertLess(size, 5000) 277 | 278 | def test_node_size(self): 279 | sl = SkipList() 280 | for i in range(1000): 281 | sl.insert(i, None) 282 | size = getsize(sl) 283 | self.assertIsInstance(size, int) 284 | self.assertGreater(size, 0) 285 | self.assertLess(size/1000, 250) 286 | 287 | def test_dump(self): 288 | sl = SkipList() 289 | sl.insert('foo', 'bar') 290 | sl.insert('baz', 'qux') 291 | out = six.StringIO() 292 | dump(sl, out) 293 | s = out.getvalue() 294 | self.assertIsInstance(s, str) 295 | self.assertGreater(len(s), 20) 296 | 297 | 298 | if __name__ == '__main__': 299 | unittest.main() 300 | -------------------------------------------------------------------------------- /pyskiplist/skiplist.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file is part of Bluepass. Bluepass is Copyright (c) 2012-2014 3 | # Geert Jansen. 4 | # 5 | # Bluepass is free software available under the GNU General Public License, 6 | # version 3. See the file LICENSE distributed with this file for the exact 7 | # licensing terms. 8 | 9 | from __future__ import absolute_import, print_function 10 | 11 | import os 12 | import sys 13 | import math 14 | import random 15 | 16 | __all__ = ['SkipList'] 17 | 18 | 19 | # The following functions are debugging functions. They are available only when 20 | # Python is not started with -O. 21 | 22 | if __debug__: 23 | 24 | def fmtnode(node): 25 | """Format a single skiplist node.""" 26 | level = max(1, len(node) - 3) 27 | skip = '(none)' if level == 1 else node[-1] 28 | return '' \ 29 | .format(level, node[0], node[1], skip) 30 | 31 | def dump(sl, file=sys.stdout): 32 | """Dump a skiplist to standard output.""" 33 | print('== Dumping skiplist {0!r}'.format(sl), file=file) 34 | print('Level: {}/{}'.format(sl.level, sl.maxlevel), file=file) 35 | print('Size: {}'.format(len(sl)), file=file) 36 | node = sl._head 37 | print('{0} (head)'.format(fmtnode(node)), file=file) 38 | node = node[2] 39 | avglvl = avglen = avgsiz = 0 40 | while node is not sl._tail: 41 | print('{0}'.format(fmtnode(node)), file=file) 42 | node = node[2] 43 | avglvl += max(1, len(node) - 3) 44 | avglen += len(node) 45 | avgsiz += nodesize(node) 46 | print('{0} (tail)'.format(fmtnode(node)), file=file) 47 | print('Avg level: {:.2f}'.format(avglvl/len(sl)), file=file) 48 | print('Avg node len: {:.2f}'.format(avglen/len(sl)), file=file) 49 | print('Avg node memory: {:.2f}'.format(avgsiz/len(sl)), file=file) 50 | print(file=file) 51 | 52 | def check(sl): 53 | """Check the internal structure of a skiplist.""" 54 | level = sl.maxlevel 55 | assert level > 0 56 | while sl._head[1+level] is sl._tail and level > 1: 57 | level -= 1 58 | assert level == sl.level 59 | assert sl._head[0] is sl._head[1] is None 60 | assert sl._head[-1] == 0 61 | pos = 0 62 | node = sl._head 63 | inbound = {id(sl._head): 0, id(sl._tail): len(sl)} 64 | while node is not sl._tail: 65 | assert isinstance(node, list) 66 | level = min(sl.level, max(1, len(node)-3)) 67 | assert 1 <= level <= sl.maxlevel 68 | for i in range(1, level): 69 | fnode = node[2+i] 70 | flevel = min(sl.level, max(1, len(fnode)-3)) 71 | if i == flevel-1: 72 | inbound[id(fnode)] = pos 73 | if level > 1: 74 | assert id(node) in inbound 75 | assert pos == inbound[id(node)] + node[-1] 76 | for i in range(level): 77 | fnode = node[2+i] 78 | assert isinstance(fnode, list) 79 | level = max(1, len(node) - 3) 80 | assert level >= i+1 81 | node = node[2] 82 | pos += 1 83 | assert sl._tail[0] is None 84 | assert sl._tail[1] is None 85 | for i in range(sl.maxlevel): 86 | assert sl._tail[2+i] is None 87 | assert len(sl) == inbound[id(sl._tail)] + node[-1] 88 | 89 | def nodesize(node): 90 | """Return the size of a skiplist node.""" 91 | size = sys.getsizeof(node) 92 | size += sys.getsizeof(node[0]) 93 | size += sys.getsizeof(node[1]) 94 | # All elements in [3:-1] are references so don't count 95 | if len(node) > 3: 96 | size += sys.getsizeof(node[-1]) 97 | return size 98 | 99 | def getsize(sl): 100 | """Return total size of a skiplist.""" 101 | size = sys.getsizeof(sl) 102 | size += sys.getsizeof(sl._level) 103 | node = sl._head 104 | while node is not sl._tail: 105 | size += nodesize(node) 106 | node = node[2] 107 | size += nodesize(node) 108 | size += sys.getsizeof(sl._path) # contains references or None 109 | size += sys.getsizeof(sl._distance) 110 | for el in sl._distance: 111 | size += sys.getsizeof(el) 112 | return size 113 | 114 | 115 | class SkipList(object): 116 | """An indexable skip list. 117 | 118 | A SkipList provides an ordered sequence of key-value pairs. The list is 119 | always sorted on key and supports O(1) forward iteration. It has O(log N) 120 | time complexity for key lookup, pair insertion and pair removal anywhere in 121 | the list. The list also supports O(log N) element access by position. 122 | 123 | The keys of all pairs you add to the skiplist must be be comparable against 124 | each other, and define the ``<`` and ``<=`` operators. 125 | """ 126 | 127 | UNSET = object() 128 | 129 | p = int((1<<31) / math.e) 130 | maxlevel = 20 131 | 132 | # Kudos to http://pythonsweetness.tumblr.com/post/45227295342 for some 133 | # useful tricks, including using a list for the nodes to save memory. 134 | 135 | # Use the built-in Mersenne Twister random number generator. It is more 136 | # appropriate than SystemRandom because we don't need cryptographically 137 | # secure random numbers, and we don't want to do a system call to read 138 | # /dev/urandom for each random number we need (every insertion needs a new 139 | # random number). 140 | 141 | _rnd = random.Random() 142 | _rnd.seed(os.urandom(16)) 143 | 144 | __slots__ = ('_level', '_head', '_tail', '_path', '_distance') 145 | 146 | def __init__(self): 147 | self._level = 1 148 | self._head = self._new_node(self.maxlevel, None, None) 149 | self._tail = self._new_node(self.maxlevel, None, None) 150 | for i in range(self.maxlevel): 151 | self._head[2+i] = self._tail 152 | self._path = [None] * self.maxlevel 153 | self._distance = [None] * self.maxlevel 154 | 155 | def _new_node(self, level, key, value): 156 | # Node layout: [key, value, next*LEVEL, skip?] 157 | # The "skip" element indicates how many nodes are skipped by the 158 | # highest level incoming link. 159 | if level == 1: 160 | return [key, value, None] 161 | else: 162 | return [key, value] + [None]*level + [0] 163 | 164 | def _random_level(self): 165 | # Exponential distribution as per Pugh's paper. 166 | l = 1 167 | maxlevel = min(self.maxlevel, self.level+1) 168 | while l < maxlevel and self._rnd.getrandbits(31) < self.p: 169 | l += 1 170 | return l 171 | 172 | def _create_node(self, key, value): 173 | # Create a new node, updating the list level if required. 174 | level = self._random_level() 175 | if level > self.level: 176 | self._tail[-1] = len(self) 177 | self._level = level 178 | self._path[level-1] = self._head 179 | self._distance[level-1] = 0 180 | return self._new_node(level, key, value) 181 | 182 | def _find_lt(self, key): 183 | # Find path to last node < key 184 | node = self._head 185 | distance = 0 186 | for i in reversed(range(self.level)): 187 | nnode = node[2+i] 188 | while nnode is not self._tail and nnode[0] < key: 189 | nnode, node = nnode[2+i], nnode 190 | distance += 1 if i == 0 else node[-1] 191 | self._path[i] = node 192 | self._distance[i] = distance 193 | 194 | def _find_lte(self, key): 195 | # Find path to last node <= key 196 | node = self._head 197 | distance = 0 198 | for i in reversed(range(self.level)): 199 | nnode = node[2+i] 200 | while nnode is not self._tail and nnode[0] <= key: 201 | nnode, node = nnode[2+i], nnode 202 | distance += 1 if i == 0 else node[-1] 203 | self._path[i] = node 204 | self._distance[i] = distance 205 | 206 | def _find_pos(self, pos): 207 | # Create path to node at pos. 208 | node = self._head 209 | distance = 0 210 | for i in reversed(range(self.level)): 211 | nnode = node[2+i] 212 | ndistance = distance + (1 if i == 0 else nnode[-1]) 213 | while nnode is not self._tail and ndistance <= pos: 214 | nnode, node, distance = nnode[2+i], nnode, ndistance 215 | ndistance += 1 if i == 0 else nnode[-1] 216 | self._path[i] = node 217 | self._distance[i] = distance 218 | 219 | def _insert(self, node): 220 | # Insert a node in the list. The _path and _distance must be set. 221 | path, distance = self._path, self._distance 222 | # Update pointers 223 | level = max(1, len(node) - 3) 224 | for i in range(level): 225 | node[2+i] = path[i][2+i] 226 | path[i][2+i] = node 227 | if level > 1: 228 | node[-1] = 1 + distance[0] - distance[level-1] 229 | # Update skip counts 230 | node = node[2] 231 | i = 2; j = min(len(node) - 3, self.level) 232 | while i <= self.level: 233 | while j < i: 234 | node = node[i] 235 | j = min(len(node) - 3, self.level) 236 | node[-1] -= distance[0] - distance[j-1] if j <= level else -1 237 | i = j+1 238 | 239 | def _remove(self, node): 240 | # Remove a node. The _path and _distance must be set. 241 | path, distance = self._path, self._distance 242 | level = max(1, len(node) - 3) 243 | for i in range(level): 244 | path[i][2+i] = node[2+i] 245 | # Update skip counts 246 | value = node[1] 247 | node = node[2] 248 | i = 2; j = min(len(node) - 3, self.level) 249 | while i <= self.level: 250 | while j < i: 251 | node = node[i] 252 | j = min(len(node) - 3, self.level) 253 | node[-1] += distance[0] - distance[j-1] if j <= level else -1 254 | i = j+1 255 | # Reduce level if last node on current level was removed 256 | while self.level > 1 and self._head[1+self.level] is self._tail: 257 | self._level -= 1 258 | self._tail[-1] += self._tail[-1] - len(self) 259 | return value 260 | 261 | # PUBLIC API ... 262 | 263 | @property 264 | def level(self): 265 | """The current level of the skip list.""" 266 | return self._level 267 | 268 | def insert(self, key, value): 269 | """Insert a key-value pair in the list. 270 | 271 | The pair is inserted at the correct location so that the list remains 272 | sorted on *key*. If a pair with the same key is already in the list, 273 | then the pair is appended after all other pairs with that key. 274 | """ 275 | self._find_lte(key) 276 | node = self._create_node(key, value) 277 | self._insert(node) 278 | 279 | def replace(self, key, value): 280 | """Replace the value of the first key-value pair with key *key*. 281 | 282 | If the key was not found, the pair is inserted. 283 | """ 284 | self._find_lt(key) 285 | node = self._path[0][2] 286 | if node is self._tail or key < node[0]: 287 | node = self._create_node(key, value) 288 | self._insert(node) 289 | else: 290 | node[1] = value 291 | 292 | def clear(self): 293 | """Remove all key-value pairs.""" 294 | for i in range(self.maxlevel): 295 | self._head[2+i] = self._tail 296 | self._tail[-1] = 0 297 | self._level = 1 298 | 299 | def __len__(self): 300 | """Return the number of pairs in the list.""" 301 | dist = 0 302 | idx = self.level + 1 303 | node = self._head[idx] 304 | while node is not self._tail: 305 | dist += node[-1] if idx > 2 else 1 306 | node = node[idx] 307 | dist += node[-1] 308 | return dist 309 | 310 | __bool__ = __nonzero__ = lambda self: len(self) > 0 311 | 312 | def __repr__(self): 313 | return type(self).__name__ + '((' + repr(list(self.items()))[1:-1] + '))' 314 | 315 | def items(self, start=None, stop=None): 316 | """Return an iterator yielding pairs. 317 | 318 | If *start* is specified, iteration starts at the first pair with a key 319 | that is larger than or equal to *start*. If not specified, iteration 320 | starts at the first pair in the list. 321 | 322 | If *stop* is specified, iteration stops at the last pair that is 323 | smaller than *stop*. If not specified, iteration end with the last pair 324 | in the list. 325 | """ 326 | if start is None: 327 | node = self._head[2] 328 | else: 329 | self._find_lt(start) 330 | node = self._path[0][2] 331 | while node is not self._tail and (stop is None or node[0] < stop): 332 | yield (node[0], node[1]) 333 | node = node[2] 334 | 335 | __iter__ = items 336 | 337 | def keys(self, start=None, stop=None): 338 | """Like :meth:`items` but returns only the keys.""" 339 | return (item[0] for item in self.items(start, stop)) 340 | 341 | def values(self, start=None, stop=None): 342 | """Like :meth:`items` but returns only the values.""" 343 | return (item[1] for item in self.items(start, stop)) 344 | 345 | def popitem(self): 346 | """Removes the first key-value pair and return it. 347 | 348 | This method raises a ``KeyError`` if the list is empty. 349 | """ 350 | node = self._head[2] 351 | if node is self._tail: 352 | raise KeyError('list is empty') 353 | self._find_lt(node[0]) 354 | self._remove(node) 355 | return (node[0], node[1]) 356 | 357 | # BY KEY API ... 358 | 359 | def search(self, key, default=None): 360 | """Find the first key-value pair with key *key* and return its value. 361 | 362 | If the key was not found, return *default*. If no default was provided, 363 | return ``None``. This method never raises a ``KeyError``. 364 | """ 365 | self._find_lt(key) 366 | node = self._path[0][2] 367 | if node is self._tail or key < node[0]: 368 | return default 369 | return node[1] 370 | 371 | def remove(self, key): 372 | """Remove the first key-value pair with key *key*. 373 | 374 | If the key was not found, a ``KeyError`` is raised. 375 | """ 376 | self._find_lt(key) 377 | node = self._path[0][2] 378 | if node is self._tail or key < node[0]: 379 | raise KeyError('{!r} is not in list'.format(key)) 380 | self._remove(node) 381 | 382 | def pop(self, key, default=UNSET): 383 | """Remove the first key-value pair with key *key*. 384 | 385 | If a pair was removed, return its value. Otherwise if *default* was 386 | provided, return *default*. Otherwise a ``KeyError`` is raised. 387 | """ 388 | self._find_lt(key) 389 | node = self._path[0][2] 390 | if node is self._tail or key < node[0]: 391 | if default is self.UNSET: 392 | raise KeyError('key {!r} not in list') 393 | return default 394 | self._remove(node) 395 | return node[1] 396 | 397 | def __contains__(self, key): 398 | """Return whether *key* is contained in the list.""" 399 | self._find_lt(key) 400 | node = self._path[0][2] 401 | return node is not self._tail and not key < node[0] 402 | 403 | def index(self, key, default=UNSET): 404 | """Find the first key-value pair with key *key* and return its position. 405 | 406 | If the key is not found, return *default*. If default was not provided, 407 | raise a ``KeyError`` 408 | """ 409 | self._find_lt(key) 410 | node = self._path[0][2] 411 | if node is self._tail or key < node[0]: 412 | if default is self.UNSET: 413 | raise KeyError('key {!r} not in list'.format(key)) 414 | return default 415 | return self._distance[0] 416 | 417 | def count(self, key): 418 | """Return the number of pairs with key *key*.""" 419 | count = 0 420 | pos = self.index(key, -1) 421 | if pos == -1: 422 | return count 423 | count += 1 424 | for i in range(pos+1, len(self)): 425 | if self[i][0] != key: 426 | break 427 | count += 1 428 | return count 429 | 430 | # BY POSITION API ... 431 | 432 | def __getitem__(self, pos): 433 | """Return a pair by its position. 434 | 435 | If *pos* is a slice, then return a generator that yields pairs as 436 | specified by the slice. 437 | """ 438 | size = len(self) 439 | if isinstance(pos, int): 440 | if pos < 0: 441 | pos += size 442 | if not 0 <= pos < size: 443 | raise IndexError('list index out of range') 444 | self._find_pos(pos) 445 | node = self._path[0][2] 446 | return (node[0], node[1]) 447 | elif isinstance(pos, slice): 448 | start, stop = pos.start, pos.stop 449 | if start is None: 450 | start = 0 451 | elif start < 0: 452 | start += size 453 | if stop is None: 454 | stop = size 455 | elif stop < 0: 456 | stop += size 457 | self._find_pos(start) 458 | def genpairs(): 459 | pos = start; node = self._path[0][2] 460 | while node is not self._tail and pos < stop: 461 | yield (node[0], node[1]) 462 | node = node[2]; pos += 1 463 | return genpairs() 464 | else: 465 | raise TypeError('expecting int or slice, got {0.__name__!r}'.format(type(pos))) 466 | 467 | def __delitem__(self, pos): 468 | """Delete a pair by its position.""" 469 | if not isinstance(pos, int): 470 | raise TypeError('expecting int, got {0.__name__!r}'.format(type(pos))) 471 | size = len(self) 472 | if pos < 0: 473 | pos += size 474 | if not 0 <= pos < size: 475 | raise IndexError('list index out of range') 476 | self._find_pos(pos) 477 | node = self._path[0][2] 478 | self._remove(node) 479 | 480 | def __setitem__(self, pos, value): 481 | """Set a value by its position.""" 482 | if not isinstance(pos, int): 483 | raise TypeError('expecting int, got {0.__name__!r}'.format(type(pos))) 484 | size = len(self) 485 | if pos < 0: 486 | pos += size 487 | if not 0 <= pos < size: 488 | raise IndexError('list index out of range') 489 | self._find_pos(pos) 490 | node = self._path[0][2] 491 | node[1] = value 492 | --------------------------------------------------------------------------------