├── hiredis ├── py.typed ├── version.py ├── __init__.py └── hiredis.pyi ├── benchmark ├── simple ├── lrange └── benchmark.py ├── setup.cfg ├── .gitignore ├── .gitmodules ├── src ├── pack.h ├── hiredis.h ├── reader.h ├── hiredis.c ├── pack.c └── reader.c ├── MANIFEST.in ├── dev_requirements.txt ├── .github ├── workflows │ ├── pypi-publish.yaml │ ├── spellcheck.yml │ ├── wheelbuild-labeled.yaml │ ├── installtest.sh │ ├── release-drafter.yml │ ├── freebsd.yaml │ ├── integration.yaml │ └── REUSABLE-wheeler.yaml ├── spellcheck-settings.yml ├── wordlist.txt └── release-drafter-config.yml ├── tests ├── test_gc.py ├── test_pack.py └── test_reader.py ├── tox.ini ├── LICENSE ├── CHANGELOG.md ├── setup.py └── README.md /hiredis/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/simple: -------------------------------------------------------------------------------- 1 | set key value 2 | get key 3 | -------------------------------------------------------------------------------- /hiredis/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "3.4.0-dev" 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /benchmark/lrange: -------------------------------------------------------------------------------- 1 | lpush list value 2 | ltrim list 0 99 3 | lrange list 0 99 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /build 3 | /dist 4 | MANIFEST 5 | .venv 6 | **/*.so 7 | hiredis.egg-info 8 | .idea 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/hiredis"] 2 | path = vendor/hiredis 3 | url = https://github.com/redis/hiredis 4 | -------------------------------------------------------------------------------- /src/pack.h: -------------------------------------------------------------------------------- 1 | #ifndef __PACK_H 2 | #define __PACK_H 3 | 4 | #include 5 | 6 | extern PyObject* pack_command(PyObject* cmd); 7 | 8 | #endif -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include MANIFEST.in 3 | include README.md 4 | include src/*.[ch] 5 | include vendor/hiredis/COPYING 6 | include vendor/hiredis/*.[ch] 7 | graft tests 8 | global-exclude __pycache__ 9 | global-exclude *.py[co] 10 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | black==24.3.0 2 | flake8==4.0.1 3 | isort==5.10.1 4 | pytest>=7.2.0 5 | memray==1.17.1 ; sys_platform != 'win32' and platform_python_implementation != 'PyPy' 6 | pytest-memray==1.7.0 ; sys_platform != 'win32' and platform_python_implementation != 'PyPy' 7 | setuptools 8 | tox==3.24.4 9 | vulture>=2.3.0 10 | wheel>=0.30.0 -------------------------------------------------------------------------------- /hiredis/__init__.py: -------------------------------------------------------------------------------- 1 | from hiredis.hiredis import Reader, HiredisError, pack_command, ProtocolError, ReplyError, PushNotification 2 | from hiredis.version import __version__ 3 | 4 | __all__ = [ 5 | "Reader", 6 | "HiredisError", 7 | "pack_command", 8 | "ProtocolError", 9 | "PushNotification", 10 | "ReplyError", 11 | "__version__"] 12 | -------------------------------------------------------------------------------- /.github/workflows/pypi-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish tag to Pypi 2 | on: 3 | workflow_dispatch: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+' 7 | permissions: 8 | contents: read # to fetch code (actions/checkout) 9 | 10 | 11 | jobs: 12 | 13 | release: 14 | uses: ./.github/workflows/REUSABLE-wheeler.yaml 15 | with: 16 | release: true 17 | secrets: inherit 18 | -------------------------------------------------------------------------------- /.github/workflows/spellcheck.yml: -------------------------------------------------------------------------------- 1 | name: spellcheck 2 | on: 3 | pull_request: 4 | jobs: 5 | check-spelling: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Checkout 9 | uses: actions/checkout@v3 10 | - name: Check Spelling 11 | uses: rojopolis/spellcheck-github-actions@0.33.1 12 | with: 13 | config_path: .github/spellcheck-settings.yml 14 | task_name: Markdown 15 | -------------------------------------------------------------------------------- /.github/workflows/wheelbuild-labeled.yaml: -------------------------------------------------------------------------------- 1 | name: Wheel builder 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - labeled 7 | 8 | permissions: 9 | contents: read # to fetch code (actions/checkout) 10 | 11 | jobs: 12 | 13 | wheelbuilder: 14 | uses: ./.github/workflows/REUSABLE-wheeler.yaml 15 | if: github.event.label.name == 'test-wheels' 16 | with: 17 | release: false 18 | secrets: inherit 19 | -------------------------------------------------------------------------------- /tests/test_gc.py: -------------------------------------------------------------------------------- 1 | import gc 2 | 3 | import hiredis 4 | 5 | 6 | def test_reader_gc(): 7 | class A: 8 | def __init__(self): 9 | self.reader = hiredis.Reader(replyError=self.reply_error) 10 | 11 | def reply_error(self, error): 12 | return Exception() 13 | 14 | A() 15 | gc.collect() 16 | 17 | assert not any(isinstance(o, A) for o in gc.get_objects()), "Referent was not collected" 18 | -------------------------------------------------------------------------------- /.github/spellcheck-settings.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | - name: Markdown 3 | expect_match: false 4 | apsell: 5 | lang: en 6 | d: en_US 7 | ignore-case: true 8 | dictionary: 9 | wordlists: 10 | - .github/wordlist.txt 11 | output: wordlist.dic 12 | pipeline: 13 | - pyspelling.filters.markdown: 14 | markdown_extensions: 15 | - markdown.extensions.extra: 16 | - pyspelling.filters.html: 17 | comments: false 18 | attributes: 19 | - alt 20 | ignores: 21 | - ':matches(code, pre)' 22 | - code 23 | - pre 24 | - blockquote 25 | - img 26 | sources: 27 | - 'README.md' 28 | - 'FAQ.md' 29 | - 'docs/**' 30 | -------------------------------------------------------------------------------- /.github/wordlist.txt: -------------------------------------------------------------------------------- 1 | ACLs 2 | Autoloading 3 | CAS 4 | Customizable 5 | ElastiCache 6 | FPM 7 | IANA 8 | Kops 9 | LRANGE 10 | Lua 11 | Misspelled words: 12 | PSR 13 | Packagist 14 | PhpRedis 15 | Predis 16 | PyPI 17 | README 18 | Redis 19 | SHA 20 | SSL 21 | TCP 22 | TLS 23 | URI 24 | autoload 25 | autoloader 26 | autoloading 27 | backend 28 | backends 29 | behaviour 30 | benchmarking 31 | boolean 32 | customizable 33 | dataset 34 | de 35 | dedcoded 36 | extensibility 37 | gcc 38 | gevent 39 | hiredis 40 | keyspace 41 | keyspaces 42 | localhost 43 | namespace 44 | pipelining 45 | pluggable 46 | py 47 | rebalanced 48 | rebalancing 49 | redis 50 | runtime 51 | sharding 52 | stunnel 53 | submodules 54 | variadic 55 | -------------------------------------------------------------------------------- /.github/workflows/installtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SUFFIX=$1 6 | if [ -z ${SUFFIX} ]; then 7 | echo "Supply valid python package extension such as whl or tar.gz. Exiting." 8 | exit 3 9 | fi 10 | 11 | script=`pwd`/${BASH_SOURCE[0]} 12 | HERE=`dirname ${script}` 13 | ROOT=`realpath ${HERE}/../..` 14 | 15 | cd ${ROOT} 16 | DESTENV=${ROOT}/.venvforinstall 17 | if [ -d ${DESTENV} ]; then 18 | rm -rf ${DESTENV} 19 | fi 20 | python -m venv ${DESTENV} 21 | source ${DESTENV}/bin/activate 22 | pip install --upgrade --quiet pip 23 | pip install --quiet -r dev_requirements.txt 24 | python setup.py sdist bdist_wheel 25 | 26 | PKG=`ls ${ROOT}/dist/*.${SUFFIX}` 27 | ls -l ${PKG} 28 | 29 | # install, run tests 30 | pip install ${PKG} -------------------------------------------------------------------------------- /src/hiredis.h: -------------------------------------------------------------------------------- 1 | #ifndef __HIREDIS_PY_H 2 | #define __HIREDIS_PY_H 3 | 4 | #include 5 | #include 6 | 7 | #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ 8 | #define PyMODINIT_FUNC void 9 | #endif 10 | 11 | #ifndef MOD_HIREDIS 12 | #define MOD_HIREDIS "hiredis" 13 | #endif 14 | 15 | struct hiredis_ModuleState { 16 | PyObject *HiErr_Base; 17 | PyObject *HiErr_ProtocolError; 18 | PyObject *HiErr_ReplyError; 19 | }; 20 | 21 | #define GET_STATE(__s) ((struct hiredis_ModuleState*)PyModule_GetState(__s)) 22 | 23 | /* Keep pointer around for other classes to access the module state. */ 24 | extern PyObject *mod_hiredis; 25 | #define HIREDIS_STATE (GET_STATE(mod_hiredis)) 26 | 27 | PyMODINIT_FUNC PyInit_hiredis(void); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 3.2.0 3 | requires = tox-docker 4 | envlist = linters 5 | 6 | [isort] 7 | profile = black 8 | multi_line_output = 3 9 | 10 | [testenv:linters] 11 | deps_files = dev_requirements.txt 12 | docker = 13 | commands = 14 | flake8 15 | black --target-version py37 --check --diff . 16 | isort --check-only --diff . 17 | vulture redis whitelist.py --min-confidence 80 18 | skipsdist = true 19 | skip_install = true 20 | 21 | 22 | [flake8] 23 | max-line-length = 88 24 | exclude = 25 | *.egg-info, 26 | *.pyc, 27 | .git, 28 | .tox, 29 | .venv*, 30 | build, 31 | docs/*, 32 | dist, 33 | docker, 34 | venv*, 35 | .venv*, 36 | whitelist.py 37 | ignore = 38 | F405 39 | W503 40 | E203 41 | E126 42 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | # branches to consider in the event; optional, defaults to all 6 | branches: 7 | - master 8 | 9 | jobs: 10 | update_release_draft: 11 | permissions: 12 | pull-requests: write # to add label to PR (release-drafter/release-drafter) 13 | contents: write # to create a github release (release-drafter/release-drafter) 14 | 15 | runs-on: ubuntu-latest 16 | steps: 17 | # Drafts your next Release notes as Pull Requests are merged into "master" 18 | - uses: release-drafter/release-drafter@v5 19 | with: 20 | # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml 21 | config-name: release-drafter-config.yml 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | -------------------------------------------------------------------------------- /src/reader.h: -------------------------------------------------------------------------------- 1 | #ifndef __READER_H 2 | #define __READER_H 3 | 4 | #include "hiredis.h" 5 | 6 | typedef struct { 7 | PyObject_HEAD 8 | redisReader *reader; 9 | char *encoding; 10 | char *errors; 11 | int shouldDecode; 12 | PyObject *protocolErrorClass; 13 | PyObject *replyErrorClass; 14 | PyObject *notEnoughDataObject; 15 | 16 | /* Stores error object in between incomplete calls to #gets, in order to 17 | * only set the error once a full reply has been read. Otherwise, the 18 | * reader could get in an inconsistent state. */ 19 | struct { 20 | PyObject *ptype; 21 | PyObject *pvalue; 22 | PyObject *ptraceback; 23 | } error; 24 | } hiredis_ReaderObject; 25 | 26 | typedef struct { 27 | PyListObject list; 28 | } PushNotificationObject; 29 | 30 | extern PyTypeObject hiredis_ReaderType; 31 | extern PyTypeObject PushNotificationType; 32 | extern redisReplyObjectFunctions hiredis_ObjectFunctions; 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /.github/release-drafter-config.yml: -------------------------------------------------------------------------------- 1 | name-template: '$NEXT_MINOR_VERSION' 2 | tag-template: 'v$NEXT_MINOR_VERSION' 3 | autolabeler: 4 | - label: 'maintenance' 5 | files: 6 | - '*.md' 7 | - '.github/*' 8 | - label: 'bug' 9 | branch: 10 | - '/bug-.+' 11 | - label: 'maintenance' 12 | branch: 13 | - '/maintenance-.+' 14 | - label: 'feature' 15 | branch: 16 | - '/feature-.+' 17 | categories: 18 | - title: 'Breaking Changes' 19 | labels: 20 | - 'breakingchange' 21 | - title: '🧪 Experimental Features' 22 | labels: 23 | - 'experimental' 24 | - title: '🚀 New Features' 25 | labels: 26 | - 'feature' 27 | - 'enhancement' 28 | - title: '🐛 Bug Fixes' 29 | labels: 30 | - 'fix' 31 | - 'bugfix' 32 | - 'bug' 33 | - 'BUG' 34 | - title: '🧰 Maintenance' 35 | label: 'maintenance' 36 | change-template: '- $TITLE (#$NUMBER)' 37 | exclude-labels: 38 | - 'skip-changelog' 39 | template: | 40 | # Changes 41 | 42 | $CHANGES 43 | 44 | ## Contributors 45 | We'd like to thank all the contributors who worked on this release! 46 | 47 | $CONTRIBUTORS 48 | 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022-2023, the maintainers of this library. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /hiredis/hiredis.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional, Union, Tuple 2 | 3 | 4 | class HiredisError(Exception): 5 | ... 6 | 7 | 8 | class ProtocolError(HiredisError): 9 | ... 10 | 11 | 12 | class ReplyError(HiredisError): 13 | ... 14 | 15 | 16 | class PushNotification(list): 17 | ... 18 | 19 | 20 | class Reader: 21 | def __init__( 22 | self, 23 | protocolError: Callable[[str], Exception] = ..., 24 | replyError: Callable[[str], Exception] = ..., 25 | encoding: Optional[str] = ..., 26 | errors: Optional[str] = ..., 27 | notEnoughData: Any = ..., 28 | ) -> None: ... 29 | 30 | def feed( 31 | self, __buf: Union[str, bytes], __off: int = ..., __len: int = ... 32 | ) -> None: ... 33 | def gets(self, __shouldDecode: bool = ...) -> Any: ... 34 | def setmaxbuf(self, __maxbuf: Optional[int]) -> None: ... 35 | def getmaxbuf(self) -> int: ... 36 | def len(self) -> int: ... 37 | def has_data(self) -> bool: ... 38 | 39 | def set_encoding( 40 | self, encoding: Optional[str] = ..., errors: Optional[str] = ... 41 | ) -> None: ... 42 | 43 | 44 | def pack_command(cmd: Tuple[str | int | float | bytes | memoryview]): ... 45 | -------------------------------------------------------------------------------- /benchmark/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys, optparse, timeit 4 | from functools import partial 5 | from redis import Redis 6 | 7 | import gevent 8 | from gevent import monkey 9 | from gevent.coros import Semaphore 10 | monkey.patch_all() 11 | 12 | parser = optparse.OptionParser() 13 | parser.add_option("-n", dest="count", metavar="COUNT", type="int", default=10000) 14 | parser.add_option("-c", dest="clients", metavar="CLIENTS", type="int", default=1), 15 | (options, args) = parser.parse_args() 16 | 17 | commands = list() 18 | for line in sys.stdin.readlines(): 19 | argv = line.strip().split() 20 | commands.append((argv[0], argv[1:])) 21 | 22 | sem = Semaphore(0) 23 | count = options.count 24 | todo = count 25 | 26 | def create_client(): 27 | global todo 28 | redis = Redis(host="localhost", port=6379, db=0) 29 | 30 | sem.acquire() 31 | while todo > 0: 32 | todo -= 1 33 | for (cmd, args) in commands: 34 | getattr(redis, cmd)(*args) 35 | 36 | def run(clients): 37 | [sem.release() for _ in range(len(clients))] 38 | 39 | # Time how long it takes for all greenlets to finish 40 | join = partial(gevent.joinall, clients) 41 | time = timeit.timeit(join, number=1) 42 | print "%.2f Kops" % ((len(commands) * count / 1000.0) / time) 43 | 44 | # Let clients connect, and kickstart benchmark a little later 45 | clients = [gevent.spawn(create_client) for _ in range(options.clients)] 46 | let = gevent.spawn(run, clients) 47 | gevent.joinall([let]) 48 | -------------------------------------------------------------------------------- /.github/workflows/freebsd.yaml: -------------------------------------------------------------------------------- 1 | name: FreeBSD 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '**/*.rst' 8 | - '**/*.md' 9 | branches: 10 | - master 11 | - '[0-9].[0-9]' 12 | pull_request: 13 | branches: 14 | - master 15 | - '[0-9].[0-9]' 16 | 17 | permissions: 18 | contents: read # to fetch code (actions/checkout) 19 | 20 | jobs: 21 | 22 | run-tests: 23 | runs-on: ubuntu-latest 24 | timeout-minutes: 30 25 | env: 26 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 27 | name: Python ${{ matrix.python-version }} FreeBSD 28 | steps: 29 | - uses: actions/checkout@v4 30 | with: 31 | submodules: recursive 32 | - name: build and test 33 | uses: vmactions/freebsd-vm@v1 34 | with: 35 | envs: 'ACTIONS_ALLOW_UNSECURE_COMMANDS' 36 | usesh: true 37 | sync: rsync 38 | copyback: false 39 | prepare: pkg install -y bash curl python39 40 | run: | 41 | curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py 42 | python3.9 get-pip.py 43 | /usr/local/bin/pip install -U pip setuptools wheel 44 | sed '5,6d' dev_requirements.txt > dev_requirements_without_memray.txt 45 | /usr/local/bin/pip install -r dev_requirements_without_memray.txt 46 | /usr/local/bin/python3.9 setup.py build_ext --inplace 47 | /usr/local/bin/python3.9 -m pytest 48 | /usr/local/bin/python3.9 setup.py bdist_wheel 49 | -------------------------------------------------------------------------------- /tests/test_pack.py: -------------------------------------------------------------------------------- 1 | import hiredis 2 | import pytest 3 | 4 | testdata = [ 5 | # all_types 6 | (('HSET', 'foo', 'key', 'value1', b'key_b', b'bytes str', 'key_mv', memoryview(b'bytes str'), b'key_i', 67, 'key_f', 3.14159265359), 7 | b'*12\r\n$4\r\nHSET\r\n$3\r\nfoo\r\n$3\r\nkey\r\n$6\r\nvalue1\r\n$5\r\nkey_b\r\n$9\r\nbytes str\r\n$6\r\nkey_mv\r\n$9\r\nbytes str\r\n$5\r\nkey_i\r\n$2\r\n67\r\n$5\r\nkey_f\r\n$13\r\n3.14159265359\r\n'), 8 | # spaces_in_cmd 9 | (('COMMAND', 'GETKEYS', 'EVAL', 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}', 2, 'key1', 'key2', 'first', 'second'), 10 | b'*9\r\n$7\r\nCOMMAND\r\n$7\r\nGETKEYS\r\n$4\r\nEVAL\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$4\r\nkey1\r\n$4\r\nkey2\r\n$5\r\nfirst\r\n$6\r\nsecond\r\n'), 11 | # null_chars 12 | (('SET', 'a', bytes(b'\xaa\x00\xffU')), 13 | b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$4\r\n\xaa\x00\xffU\r\n'), 14 | # encoding 15 | (('SET', 'a', "йדב문諺"), 16 | b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$12\r\n\xD0\xB9\xD7\x93\xD7\x91\xEB\xAC\xB8\xE8\xAB\xBA\r\n'), 17 | # big_int 18 | (('SET', 'a', 2**128), 19 | b'*3\r\n$3\r\nSET\r\n$1\r\na\r\n$39\r\n340282366920938463463374607431768211456\r\n'), 20 | ] 21 | 22 | testdata_ids = [ 23 | "all_types", 24 | "spaces_in_cmd", 25 | "null_chars", 26 | "encoding", 27 | "big_int", 28 | ] 29 | 30 | 31 | @pytest.mark.parametrize("cmd,expected_packed_cmd", testdata, ids=testdata_ids) 32 | def test_basic(cmd, expected_packed_cmd): 33 | packed_cmd = hiredis.pack_command(cmd) 34 | assert packed_cmd == expected_packed_cmd 35 | 36 | 37 | def test_wrong_type(): 38 | with pytest.raises(TypeError): 39 | hiredis.pack_command(("HSET", "foo", True)) 40 | 41 | class Bar: 42 | def __init__(self) -> None: 43 | self.key = "key" 44 | self.value = 36 45 | 46 | with pytest.raises(TypeError): 47 | hiredis.pack_command(("HSET", "foo", Bar())) 48 | -------------------------------------------------------------------------------- /.github/workflows/integration.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '**/*.rst' 8 | - '**/*.md' 9 | branches: 10 | - master 11 | - 'v[0-9].[0-9]' 12 | pull_request: 13 | branches: 14 | - master 15 | - 'v[0-9].[0-9]' 16 | 17 | permissions: 18 | contents: read # to fetch code (actions/checkout) 19 | 20 | jobs: 21 | 22 | run-tests: 23 | runs-on: ${{matrix.os}} 24 | timeout-minutes: 30 25 | strategy: 26 | max-parallel: 15 27 | matrix: 28 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy-3.8', 'pypy-3.9', 'pypy-3.10'] 29 | os: ['ubuntu-latest', 'windows-latest', 'macos-latest'] 30 | fail-fast: false 31 | env: 32 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 33 | name: Python ${{ matrix.python-version }} ${{matrix.os}} 34 | steps: 35 | - uses: actions/checkout@v3 36 | with: 37 | submodules: recursive 38 | - uses: actions/setup-python@v4 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | cache: 'pip' 42 | cache-dependency-path: dev_requirements.txt 43 | - name: Install memray deps 44 | if: ${{ matrix.os == 'ubuntu-latest' }} 45 | run: | 46 | sudo apt-get install build-essential python3-dev libdebuginfod-dev libunwind-dev liblz4-dev -y -qq 47 | - name: run tests 48 | env: 49 | MACOSX_DEPLOYMENT_TARGET: 10.14 50 | run: | 51 | pip install -U pip setuptools wheel 52 | pip install -r dev_requirements.txt 53 | python setup.py build_ext --inplace 54 | python -m pytest 55 | - name: run tests with assertions enabled 56 | if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' }} 57 | run: | 58 | python setup.py build_ext -UNDEBUG --inplace --force 59 | python -m pytest 60 | - name: build and install the wheel 61 | run: | 62 | python setup.py bdist_wheel 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Hiredis-py Changelog 2 | 3 | Since [v2.2.1](https://github.com/redis/hiredis-py/releases/tag/v2.2.1) we track changelog using [GitHub releases](https://github.com/redis/hiredis-py/releases). 4 | Below you can find the changelog for all versions prior to that. 5 | 6 | ----------------- 7 | 8 | ### 2.1.1 (2023-10-01) 9 | 10 | * Restores publishing of source distribution (#139) 11 | * Fix url in Issue tracker (#140) 12 | * Version 2.1.1 (#143) 13 | * Update CHANGELOG.md for 2.1.0 (#142) 14 | 15 | ### 2.1.0 (2022-12-14) 16 | 17 | * Supporting hiredis 1.1.0 (#135) 18 | * Modernizing: Restoring CI, Moving to pytest (#136) 19 | * Adding LICENSE to Repository (#132) 20 | * Python 3.11 trove, and links back to the project (#131) 21 | * Integrating release drafter (#133) 22 | 23 | ### 2.0.0 (2021-03-28) 24 | 25 | * Bump hiredis from 0.13.3 to 1.0.0 and consequently add support for RESP3 (see #104) 26 | * Add type hints (see #106) 27 | * Build aarch64 (arm64) wheels (see #98) 28 | * Drop support for EOL Python versions 2.7, 3.4, and 3.5 (see #103) 29 | 30 | ### 1.1.0 (2020-07-15) 31 | 32 | * Allow "encoding" and "errors" attributes to be updated at runtime (see #96) 33 | 34 | ### 1.0.1 (2019-11-13) 35 | 36 | * Permit all allowed values of codec errors (see #86) 37 | * BUGFIX: READEME.md has UTF-8 characters, setup.py will fail on systems 38 | where the locale is not UTF-8. (see #89) 39 | 40 | ### 1.0.0 (2019-01-20) 41 | 42 | * **(BREAKING CHANGE)** Add ability to control how unicode decoding errors are handled (see #82) 43 | * Removed support for EOL Python 2.6, 3.2, and 3.3. 44 | 45 | ### 0.3.1 (2018-12-24) 46 | 47 | * Include test files in sdist tarball (see #80) 48 | 49 | ### 0.3.0 (2018-11-16) 50 | 51 | * Upgrade hiredis to 0.13.3 52 | * Add optional "shouldDecode" argument to Reader.gets() (see #77) 53 | * Add a "has_data" method to Reader objects (see #78) 54 | * Fix non-utf8 reply parsing causing segmentation fault in Python 3 (see #73) 55 | * Rename `state` to `hiredis_py_module_state` to avoid conflicts (see #72) 56 | * Expose len method to retrieve the buffer length (see #61) 57 | * Fix crash when custom exception raise error (on init) (see #57) 58 | * incref before PyModule_AddObject which steals references (see #48) 59 | * Sort list of source files to allow reproducible building (see #47) 60 | 61 | ### 0.2.0 (2015-04-03) 62 | 63 | * Allow usage of setuptools 64 | * Upgrade to latest hiredis including basic Windows support 65 | * Expose hiredis maxbuf settings in python 66 | 67 | ### 0.1.6 (2015-01-28) 68 | 69 | * Updated with hiredis 0.12.1 — now only uses Redis parser, not entire library (#30). 70 | 71 | ### 0.1.5 72 | 73 | * Fix memory leak when many reader instances are created (see #26). 74 | 75 | ### 0.1.4 76 | 77 | * Allow any buffer compatible object as argument to feed (see #22). 78 | 79 | ### 0.1.3 80 | 81 | * Allow `protocolError` and `replyError` to be any type of callable (see #21). 82 | 83 | ### 0.1.2 84 | 85 | * Upgrade hiredis to 0.11.0 to support deeply nested multi bulk replies. 86 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | try: 4 | from setuptools import setup, Extension 5 | except ImportError: 6 | from distutils.core import setup, Extension 7 | import importlib 8 | import glob 9 | import io 10 | import sys 11 | 12 | 13 | def version(): 14 | loader = importlib.machinery.SourceFileLoader( 15 | "hiredis.version", "hiredis/version.py") 16 | module = loader.load_module() 17 | return module.__version__ 18 | 19 | 20 | def get_sources(): 21 | hiredis_sources = ("alloc", "async", "hiredis", "net", "read", "sds", "sockcompat") 22 | return sorted(glob.glob("src/*.c") + ["vendor/hiredis/%s.c" % src for src in hiredis_sources]) 23 | 24 | 25 | def get_linker_args(): 26 | if 'win32' in sys.platform or 'darwin' in sys.platform: 27 | return [] 28 | else: 29 | return ["-Wl,-Bsymbolic", ] 30 | 31 | 32 | def get_compiler_args(): 33 | if 'win32' in sys.platform: 34 | return [] 35 | else: 36 | return ["-std=c99"] 37 | 38 | 39 | def get_libraries(): 40 | if 'win32' in sys.platform: 41 | return ["ws2_32", ] 42 | else: 43 | return [] 44 | 45 | 46 | ext = Extension("hiredis.hiredis", 47 | sources=get_sources(), 48 | extra_compile_args=get_compiler_args(), 49 | extra_link_args=get_linker_args(), 50 | libraries=get_libraries(), 51 | include_dirs=["vendor"]) 52 | 53 | setup( 54 | name="hiredis", 55 | version=version(), 56 | description="Python wrapper for hiredis", 57 | long_description=io.open('README.md', 'rt', encoding='utf-8').read(), 58 | long_description_content_type='text/markdown', 59 | url="https://github.com/redis/hiredis-py", 60 | author="Jan-Erik Rediger, Pieter Noordhuis", 61 | author_email="janerik@fnordig.de, pcnoordhuis@gmail.com", 62 | keywords=["Redis"], 63 | license="MIT", 64 | packages=["hiredis"], 65 | package_data={"hiredis": ["hiredis.pyi", "py.typed"]}, 66 | ext_modules=[ext], 67 | python_requires=">=3.8", 68 | project_urls={ 69 | "Changes": "https://github.com/redis/hiredis-py/releases", 70 | "Issue tracker": "https://github.com/redis/hiredis-py/issues", 71 | }, 72 | classifiers=[ 73 | 'Development Status :: 5 - Production/Stable', 74 | 'Intended Audience :: Developers', 75 | 'License :: OSI Approved :: MIT License', 76 | 'Operating System :: MacOS', 77 | 'Operating System :: POSIX', 78 | 'Programming Language :: C', 79 | 'Programming Language :: Python :: 3', 80 | 'Programming Language :: Python :: 3 :: Only', 81 | 'Programming Language :: Python :: 3.8', 82 | 'Programming Language :: Python :: 3.9', 83 | 'Programming Language :: Python :: 3.10', 84 | 'Programming Language :: Python :: 3.11', 85 | 'Programming Language :: Python :: 3.12', 86 | 'Programming Language :: Python :: 3.13', 87 | 'Programming Language :: Python :: 3.14', 88 | 'Programming Language :: Python :: Implementation :: CPython', 89 | 'Topic :: Software Development', 90 | ], 91 | ) 92 | -------------------------------------------------------------------------------- /src/hiredis.c: -------------------------------------------------------------------------------- 1 | #include "hiredis.h" 2 | #include "reader.h" 3 | #include "pack.h" 4 | 5 | static int hiredis_ModuleTraverse(PyObject *m, visitproc visit, void *arg) { 6 | Py_VISIT(GET_STATE(m)->HiErr_Base); 7 | Py_VISIT(GET_STATE(m)->HiErr_ProtocolError); 8 | Py_VISIT(GET_STATE(m)->HiErr_ReplyError); 9 | return 0; 10 | } 11 | 12 | static int hiredis_ModuleClear(PyObject *m) { 13 | Py_CLEAR(GET_STATE(m)->HiErr_Base); 14 | Py_CLEAR(GET_STATE(m)->HiErr_ProtocolError); 15 | Py_CLEAR(GET_STATE(m)->HiErr_ReplyError); 16 | return 0; 17 | } 18 | 19 | static PyObject* 20 | py_pack_command(PyObject* self, PyObject* cmd) 21 | { 22 | return pack_command(cmd); 23 | } 24 | 25 | PyDoc_STRVAR(pack_command_doc, "Pack a series of arguments into the Redis protocol"); 26 | 27 | PyMethodDef pack_command_method = { 28 | "pack_command", /* The name as a C string. */ 29 | (PyCFunction) py_pack_command, /* The C function to invoke. */ 30 | METH_O, /* Flags telling Python how to invoke */ 31 | pack_command_doc, /* The docstring as a C string. */ 32 | }; 33 | 34 | 35 | PyMethodDef methods[] = { 36 | {"pack_command", (PyCFunction) py_pack_command, METH_O, pack_command_doc}, 37 | {NULL}, 38 | }; 39 | 40 | static struct PyModuleDef hiredis_ModuleDef = { 41 | PyModuleDef_HEAD_INIT, 42 | MOD_HIREDIS, 43 | NULL, 44 | sizeof(struct hiredis_ModuleState), /* m_size */ 45 | methods, /* m_methods */ 46 | NULL, /* m_reload */ 47 | hiredis_ModuleTraverse, /* m_traverse */ 48 | hiredis_ModuleClear, /* m_clear */ 49 | NULL /* m_free */ 50 | }; 51 | 52 | /* Keep pointer around for other classes to access the module state. */ 53 | PyObject *mod_hiredis; 54 | 55 | PyMODINIT_FUNC PyInit_hiredis(void) 56 | 57 | { 58 | if (PyType_Ready(&hiredis_ReaderType) < 0) { 59 | return NULL; 60 | } 61 | 62 | PushNotificationType.tp_base = &PyList_Type; 63 | if (PyType_Ready(&PushNotificationType) < 0) { 64 | return NULL; 65 | } 66 | 67 | mod_hiredis = PyModule_Create(&hiredis_ModuleDef); 68 | 69 | /* Setup custom exceptions */ 70 | HIREDIS_STATE->HiErr_Base = 71 | PyErr_NewException(MOD_HIREDIS ".HiredisError", PyExc_Exception, NULL); 72 | HIREDIS_STATE->HiErr_ProtocolError = 73 | PyErr_NewException(MOD_HIREDIS ".ProtocolError", HIREDIS_STATE->HiErr_Base, NULL); 74 | HIREDIS_STATE->HiErr_ReplyError = 75 | PyErr_NewException(MOD_HIREDIS ".ReplyError", HIREDIS_STATE->HiErr_Base, NULL); 76 | 77 | Py_INCREF(HIREDIS_STATE->HiErr_Base); 78 | PyModule_AddObject(mod_hiredis, "HiredisError", HIREDIS_STATE->HiErr_Base); 79 | Py_INCREF(HIREDIS_STATE->HiErr_ProtocolError); 80 | PyModule_AddObject(mod_hiredis, "ProtocolError", HIREDIS_STATE->HiErr_ProtocolError); 81 | Py_INCREF(HIREDIS_STATE->HiErr_ReplyError); 82 | PyModule_AddObject(mod_hiredis, "ReplyError", HIREDIS_STATE->HiErr_ReplyError); 83 | 84 | Py_INCREF(&hiredis_ReaderType); 85 | PyModule_AddObject(mod_hiredis, "Reader", (PyObject *)&hiredis_ReaderType); 86 | 87 | Py_INCREF(&PushNotificationType); 88 | PyModule_AddObject(mod_hiredis, "PushNotification", (PyObject *)&PushNotificationType); 89 | 90 | return mod_hiredis; 91 | } 92 | -------------------------------------------------------------------------------- /.github/workflows/REUSABLE-wheeler.yaml: -------------------------------------------------------------------------------- 1 | name: Wheel builder 2 | 3 | on: 4 | 5 | workflow_call: 6 | 7 | inputs: 8 | release: 9 | required: false 10 | type: boolean 11 | default: false 12 | 13 | 14 | permissions: 15 | contents: read # to fetch code (actions/checkout) 16 | 17 | jobs: 18 | 19 | build_wheels: 20 | name: Build wheels on ${{ matrix.os }} 21 | runs-on: ${{ matrix.os }} 22 | strategy: 23 | matrix: 24 | os: [ubuntu-latest, windows-latest, macos-latest] 25 | env: 26 | CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" 27 | MACOSX_DEPLOYMENT_TARGET: "10.15" 28 | steps: 29 | - uses: actions/checkout@v4 30 | with: 31 | submodules: recursive 32 | 33 | - name: Set up QEMU 34 | if: runner.os == 'Linux' 35 | uses: docker/setup-qemu-action@v3 36 | with: 37 | platforms: all 38 | 39 | - name: Build wheels 40 | uses: pypa/cibuildwheel@v3.2.0 41 | env: 42 | # configure cibuildwheel to build native archs ('auto'), and some 43 | # emulated ones 44 | CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x 45 | 46 | - uses: actions/upload-artifact@v4 47 | with: 48 | name: ${{matrix.os}}-wheels 49 | path: ./wheelhouse/*.whl 50 | 51 | build_sdist: 52 | name: Build source dist 53 | runs-on: ubuntu-latest 54 | steps: 55 | - uses: actions/setup-python@v5 56 | with: 57 | python-version: '3.10' 58 | - uses: actions/checkout@v4 59 | with: 60 | submodules: recursive 61 | - name: Build sdist 62 | run: | 63 | python3 setup.py sdist 64 | - uses: actions/upload-artifact@v4 65 | with: 66 | name: source-dist 67 | path: ./dist/*.tar.gz 68 | 69 | publish: 70 | name: Pypi publish 71 | if: ${{inputs.release == true}} 72 | needs: ['build_wheels', 'build_sdist'] 73 | runs-on: ubuntu-latest 74 | steps: 75 | - uses: actions/setup-python@v5 76 | with: 77 | python-version: '3.10' 78 | - name: Install tools 79 | run: | 80 | pip install twine wheel 81 | - uses: actions/download-artifact@v4 82 | with: 83 | name: ubuntu-latest-wheels 84 | path: artifacts/linux 85 | - uses: actions/download-artifact@v4 86 | with: 87 | name: windows-latest-wheels 88 | path: artifacts/windows 89 | - uses: actions/download-artifact@v4 90 | with: 91 | name: macos-latest-wheels 92 | path: artifacts/macos 93 | - uses: actions/download-artifact@v4 94 | with: 95 | name: source-dist 96 | path: artifacts/sdist 97 | - name: Unify wheel structure 98 | run: | 99 | mkdir dist 100 | cp -R artifacts/windows/* dist 101 | cp -R artifacts/linux/* dist 102 | cp -R artifacts/macos/* dist 103 | cp -R artifacts/sdist/* dist 104 | - name: Publish to Pypi 105 | uses: pypa/gh-action-pypi-publish@release/v1 106 | with: 107 | user: __token__ 108 | password: ${{ secrets.PYPI_API_TOKEN }} 109 | -------------------------------------------------------------------------------- /src/pack.c: -------------------------------------------------------------------------------- 1 | #include "pack.h" 2 | 3 | #ifndef _MSC_VER 4 | #include 5 | #else 6 | /* Workaround for https://bugs.python.org/issue11717. 7 | * defines ssize_t which can conflict 8 | * with Python's definition. 9 | */ 10 | extern long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); 11 | typedef char *sds; 12 | extern void sds_free(void *ptr); 13 | extern sds sdsempty(void); 14 | extern void sdsfreesplitres(sds *tokens, int count); 15 | extern sds sdscpylen(sds s, const char *t, size_t len); 16 | extern sds sdsnewlen(const void *init, size_t initlen); 17 | #endif 18 | 19 | #include 20 | 21 | PyObject * 22 | pack_command(PyObject *cmd) 23 | { 24 | assert(cmd); 25 | PyObject *result = NULL; 26 | 27 | if (cmd == NULL || !PyTuple_Check(cmd)) 28 | { 29 | PyErr_SetString(PyExc_TypeError, 30 | "The argument must be a tuple of str, int, float or bytes."); 31 | return NULL; 32 | } 33 | 34 | Py_ssize_t tokens_number = PyTuple_Size(cmd); 35 | sds *tokens = hi_malloc(sizeof(sds) * tokens_number); 36 | if (tokens == NULL) 37 | { 38 | return PyErr_NoMemory(); 39 | } 40 | 41 | memset(tokens, 0, sizeof(sds) * tokens_number); 42 | 43 | size_t *lengths = hi_malloc(sizeof(size_t) * tokens_number); 44 | if (lengths == NULL) 45 | { 46 | sds_free(tokens); 47 | return PyErr_NoMemory(); 48 | } 49 | 50 | Py_ssize_t len = 0; 51 | for (Py_ssize_t i = 0; i < PyTuple_Size(cmd); i++) 52 | { 53 | PyObject *item = PyTuple_GetItem(cmd, i); 54 | 55 | if (PyBytes_Check(item)) 56 | { 57 | char *bytes = NULL; 58 | Py_buffer buffer; 59 | PyObject_GetBuffer(item, &buffer, PyBUF_SIMPLE); 60 | PyBytes_AsStringAndSize(item, &bytes, &len); 61 | tokens[i] = sdsempty(); 62 | tokens[i] = sdscpylen(tokens[i], bytes, len); 63 | lengths[i] = buffer.len; 64 | PyBuffer_Release(&buffer); 65 | } 66 | else if (PyUnicode_Check(item)) 67 | { 68 | const char *bytes = PyUnicode_AsUTF8AndSize(item, &len); 69 | if (bytes == NULL) 70 | { 71 | // PyUnicode_AsUTF8AndSize sets an exception. 72 | goto cleanup; 73 | } 74 | 75 | tokens[i] = sdsnewlen(bytes, len); 76 | lengths[i] = len; 77 | } 78 | else if (PyMemoryView_Check(item)) 79 | { 80 | Py_buffer *p_buf = PyMemoryView_GET_BUFFER(item); 81 | tokens[i] = sdsnewlen(p_buf->buf, p_buf->len); 82 | lengths[i] = p_buf->len; 83 | } 84 | else 85 | { 86 | if (PyLong_CheckExact(item) || PyFloat_Check(item)) 87 | { 88 | PyObject *repr = PyObject_Repr(item); 89 | const char *bytes = PyUnicode_AsUTF8AndSize(repr, &len); 90 | 91 | tokens[i] = sdsnewlen(bytes, len); 92 | lengths[i] = len; 93 | Py_DECREF(repr); 94 | } 95 | else 96 | { 97 | PyErr_SetString(PyExc_TypeError, 98 | "A tuple item must be str, int, float or bytes."); 99 | goto cleanup; 100 | } 101 | } 102 | } 103 | 104 | char *resp_bytes = NULL; 105 | 106 | len = redisFormatCommandArgv(&resp_bytes, tokens_number, (const char **)tokens, lengths); 107 | 108 | if (len == -1) 109 | { 110 | PyErr_SetString(PyExc_RuntimeError, 111 | "Failed to serialize the command."); 112 | goto cleanup; 113 | } 114 | 115 | result = PyBytes_FromStringAndSize(resp_bytes, len); 116 | hi_free(resp_bytes); 117 | cleanup: 118 | sdsfreesplitres(tokens, tokens_number); 119 | hi_free(lengths); 120 | return result; 121 | } 122 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hiredis-py 2 | 3 | [![Build Status](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml/badge.svg)](https://github.com/redis/hiredis-py/actions/workflows/integration.yaml) 4 | [![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) 5 | [![pypi](https://badge.fury.io/py/hiredis.svg)](https://pypi.org/project/hiredis/) 6 | 7 | Python extension that wraps protocol parsing code in [hiredis][hiredis]. 8 | It primarily speeds up parsing of multi bulk replies. 9 | 10 | [hiredis]: http://github.com/redis/hiredis 11 | 12 | ## How do I Redis? 13 | 14 | [Learn for free at Redis University](https://university.redis.com/) 15 | 16 | [Build faster with the Redis Launchpad](https://launchpad.redis.com/) 17 | 18 | [Try the Redis Cloud](https://redis.com/try-free/) 19 | 20 | [Dive in developer tutorials](https://developer.redis.com/) 21 | 22 | [Join the Redis community](https://redis.com/community/) 23 | 24 | [Work at Redis](https://redis.com/company/careers/jobs/) 25 | 26 | ## Install 27 | 28 | hiredis-py is available on [PyPI](https://pypi.org/project/hiredis/), and can be installed via: 29 | 30 | ```bash 31 | pip install hiredis 32 | ``` 33 | ## Building and Testing 34 | 35 | Building this repository requires a recursive checkout of submodules, and building hiredis. The following example shows how to clone, compile, and run tests. Please note - you will need the gcc installed. 36 | 37 | ```bash 38 | git clone --recurse-submodules https://github.com/redis/hiredis-py 39 | python setup.py build_ext --inplace 40 | python -m pytest 41 | ``` 42 | 43 | ### Requirements 44 | 45 | hiredis-py requires **Python 3.8+**. 46 | 47 | Make sure Python development headers are available when installing hiredis-py. 48 | On Ubuntu/Debian systems, install them with `apt-get install python3-dev`. 49 | 50 | ## Usage 51 | 52 | The `hiredis` module contains the `Reader` class. This class is responsible for 53 | parsing replies from the stream of data that is read from a Redis connection. 54 | It does not contain functionality to handle I/O. 55 | 56 | ### Reply parser 57 | 58 | The `Reader` class has two methods that are used when parsing replies from a 59 | stream of data. `Reader.feed` takes a string argument that is appended to the 60 | internal buffer. `Reader.gets` reads this buffer and returns a reply when the 61 | buffer contains a full reply. If a single call to `feed` contains multiple 62 | replies, `gets` should be called multiple times to extract all replies. 63 | 64 | Example: 65 | 66 | ```python 67 | >>> reader = hiredis.Reader() 68 | >>> reader.feed("$5\r\nhello\r\n") 69 | >>> reader.gets() 70 | b'hello' 71 | ``` 72 | 73 | When the buffer does not contain a full reply, `gets` returns `False`. 74 | This means extra data is needed and `feed` should be called again before calling 75 | `gets` again. Alternatively you could provide custom sentinel object via parameter, 76 | which is useful for RESP3 protocol where native boolean types are supported: 77 | 78 | Example: 79 | 80 | ```python 81 | >>> reader.feed("*2\r\n$5\r\nhello\r\n") 82 | >>> reader.gets() 83 | False 84 | >>> reader.feed("$5\r\nworld\r\n") 85 | >>> reader.gets() 86 | [b'hello', b'world'] 87 | >>> reader = hiredis.Reader(notEnoughData=Ellipsis) 88 | >>> reader.gets() 89 | Ellipsis 90 | ``` 91 | 92 | #### Unicode 93 | 94 | `hiredis.Reader` is able to decode bulk data to any encoding Python supports. 95 | To do so, specify the encoding you want to use for decoding replies when 96 | initializing it: 97 | 98 | ```python 99 | >>> reader = hiredis.Reader(encoding="utf-8", errors="strict") 100 | >>> reader.feed(b"$3\r\n\xe2\x98\x83\r\n") 101 | >>> reader.gets() 102 | '☃' 103 | ``` 104 | 105 | Decoding of bulk data will be attempted using the specified encoding and 106 | error handler. If the error handler is `'strict'` (the default), a 107 | `UnicodeDecodeError` is raised when data cannot be dedcoded. This is identical 108 | to Python's default behavior. Other valid values to `errors` include 109 | `'replace'`, `'ignore'`, and `'backslashreplace'`. More information on the 110 | behavior of these error handlers can be found 111 | [here](https://docs.python.org/3/howto/unicode.html#the-string-type). 112 | 113 | 114 | When the specified encoding cannot be found, a `LookupError` will be raised 115 | when calling `gets` for the first reply with bulk data. 116 | 117 | #### Error handling 118 | 119 | When a protocol error occurs (because of multiple threads using the same 120 | socket, or some other condition that causes a corrupt stream), the error 121 | `hiredis.ProtocolError` is raised. Because the buffer is read in a lazy 122 | fashion, it will only be raised when `gets` is called and the first reply in 123 | the buffer contains an error. There is no way to recover from a faulty protocol 124 | state, so when this happens, the I/O code feeding data to `Reader` should 125 | probably reconnect. 126 | 127 | Redis can reply with error replies (`-ERR ...`). For these replies, the custom 128 | error class `hiredis.ReplyError` is returned, **but not raised**. 129 | 130 | When other error types should be used (so existing code doesn't have to change 131 | its `except` clauses), `Reader` can be initialized with the `protocolError` and 132 | `replyError` keywords. These keywords should contain a *class* that is a 133 | subclass of `Exception`. When not provided, `Reader` will use the default 134 | error types. 135 | 136 | ## Benchmarks 137 | 138 | The repository contains a benchmarking script in the `benchmark` directory, 139 | which uses [gevent](http://gevent.org/) to have non-blocking I/O and redis-py 140 | to handle connections. These benchmarks are done with a patched version of 141 | redis-py that uses hiredis-py when it is available. 142 | 143 | All benchmarks are done with 10 concurrent connections. 144 | 145 | * SET key value + GET key 146 | * redis-py: 11.76 Kops 147 | * redis-py *with* hiredis-py: 13.40 Kops 148 | * improvement: **1.1x** 149 | 150 | List entries in the following tests are 5 bytes. 151 | 152 | * LRANGE list 0 **9**: 153 | * redis-py: 4.78 Kops 154 | * redis-py *with* hiredis-py: 12.94 Kops 155 | * improvement: **2.7x** 156 | * LRANGE list 0 **99**: 157 | * redis-py: 0.73 Kops 158 | * redis-py *with* hiredis-py: 11.90 Kops 159 | * improvement: **16.3x** 160 | * LRANGE list 0 **999**: 161 | * redis-py: 0.07 Kops 162 | * redis-py *with* hiredis-py: 5.83 Kops 163 | * improvement: **83.2x** 164 | 165 | Throughput improvement for simple SET/GET is minimal, but the larger multi bulk replies 166 | get, the larger the performance improvement is. 167 | 168 | ## License 169 | 170 | This code is released under the BSD license, after the license of hiredis. 171 | -------------------------------------------------------------------------------- /tests/test_reader.py: -------------------------------------------------------------------------------- 1 | import hiredis 2 | import pytest 3 | 4 | @pytest.fixture() 5 | def reader(): 6 | return hiredis.Reader() 7 | 8 | # def reply(): 9 | # return reader.gets() 10 | 11 | def test_nothing(reader): 12 | assert not reader.gets() 13 | 14 | def test_error_when_feeding_non_string(reader): 15 | with pytest.raises(TypeError): 16 | reader.feed(1) 17 | 18 | def test_protocol_error(reader): 19 | reader.feed(b"x") 20 | with pytest.raises(hiredis.ProtocolError): 21 | reader.gets() 22 | 23 | def test_protocol_error_with_custom_class(): 24 | r = hiredis.Reader(protocolError=RuntimeError) 25 | r.feed(b"x") 26 | with pytest.raises(RuntimeError): 27 | r.gets() 28 | 29 | def test_protocol_error_with_custom_callable(): 30 | 31 | class CustomException(Exception): 32 | pass 33 | 34 | r = hiredis.Reader(protocolError=lambda e: CustomException(e)) 35 | r.feed(b"x") 36 | with pytest.raises(CustomException): 37 | r.gets() 38 | 39 | def test_fail_with_wrong_protocol_error_class(): 40 | with pytest.raises(TypeError): 41 | hiredis.Reader(protocolError="wrong") 42 | 43 | def test_faulty_protocol_error_class(): 44 | def make_error(errstr): 45 | 1 / 0 46 | r = hiredis.Reader(protocolError=make_error) 47 | r.feed(b"x") 48 | with pytest.raises(ZeroDivisionError): 49 | r.gets() 50 | 51 | def test_error_string(reader): 52 | reader.feed(b"-error\r\n") 53 | error = reader.gets() 54 | 55 | assert isinstance(error, hiredis.ReplyError) 56 | assert ("error", ) == error.args 57 | 58 | def test_error_string_with_custom_class(): 59 | r = hiredis.Reader(replyError=RuntimeError) 60 | r.feed(b"-error\r\n") 61 | error = r.gets() 62 | 63 | assert isinstance(error, RuntimeError) 64 | assert ("error", ) == error.args 65 | 66 | def test_error_string_with_custom_callable(): 67 | class CustomException(Exception): 68 | pass 69 | 70 | r= hiredis.Reader(replyError=lambda e: CustomException(e)) 71 | r.feed(b"-error\r\n") 72 | error = r.gets() 73 | 74 | assert isinstance(error, CustomException) 75 | assert ("error", ) == error.args 76 | 77 | def test_error_string_with_non_utf8_chars(reader): 78 | reader.feed(b"-error \xd1\r\n") 79 | error = reader.gets() 80 | 81 | expected = "error \ufffd" 82 | 83 | assert isinstance(error, hiredis.ReplyError) 84 | assert (expected,) == error.args 85 | 86 | def test_fail_with_wrong_reply_error_class(): 87 | with pytest.raises(TypeError): 88 | hiredis.Reader(replyError="wrong") 89 | 90 | def test_faulty_reply_error_class(): 91 | def make_error(errstr): 92 | 1 / 0 93 | 94 | r= hiredis.Reader(replyError=make_error) 95 | r.feed(b"-error\r\n") 96 | with pytest.raises(ZeroDivisionError): 97 | r.gets() 98 | 99 | def test_errors_in_nested_multi_bulk(reader): 100 | reader.feed(b"*2\r\n-err0\r\n-err1\r\n") 101 | 102 | for r, error in zip(("err0", "err1"), reader.gets()): 103 | assert isinstance(error, hiredis.ReplyError) 104 | assert (r,) == error.args 105 | 106 | def test_errors_with_non_utf8_chars_in_nested_multi_bulk(reader): 107 | reader.feed(b"*2\r\n-err\xd1\r\n-err1\r\n") 108 | 109 | expected = "err\ufffd" 110 | 111 | for r, error in zip((expected, "err1"), reader.gets()): 112 | assert isinstance(error, hiredis.ReplyError) 113 | assert (r,) == error.args 114 | 115 | def test_integer(reader): 116 | value = 2**63-1 # Largest 64-bit signed integer 117 | reader.feed((":%d\r\n" % value).encode("ascii")) 118 | assert value == reader.gets() 119 | 120 | def test_float(reader): 121 | value = -99.99 122 | reader.feed(b",%f\r\n" % value) 123 | assert value == reader.gets() 124 | 125 | def test_boolean_true(reader): 126 | reader.feed(b"#t\r\n") 127 | assert reader.gets() 128 | 129 | def test_boolean_false(reader): 130 | reader.feed(b"#f\r\n") 131 | assert not reader.gets() 132 | 133 | def test_none(reader): 134 | reader.feed(b"_\r\n") 135 | assert reader.gets() is None 136 | 137 | def test_set(reader): 138 | reader.feed(b"~3\r\n+tangerine\r\n_\r\n,10.5\r\n") 139 | assert [b"tangerine", None, 10.5] == reader.gets() 140 | 141 | def test_set_with_nested_dict(reader): 142 | reader.feed(b"~2\r\n+tangerine\r\n%1\r\n+a\r\n:1\r\n") 143 | assert [b"tangerine", {b"a": 1}] == reader.gets() 144 | 145 | def test_dict(reader): 146 | reader.feed(b"%2\r\n+radius\r\n,4.5\r\n+diameter\r\n:9\r\n") 147 | assert {b"radius": 4.5, b"diameter": 9} == reader.gets() 148 | 149 | @pytest.mark.limit_memory("50 KB") 150 | def test_dict_memory_leaks(reader): 151 | data = ( 152 | b"%5\r\n" 153 | b"+radius\r\n,4.5\r\n" 154 | b"+diameter\r\n:9\r\n" 155 | b"+nested_map\r\n" 156 | b"%2\r\n" 157 | b"+key1\r\n+value1\r\n" 158 | b"+key2\r\n:42\r\n" 159 | b"+nested_array\r\n" 160 | b"*2\r\n" 161 | b"+item1\r\n" 162 | b"+item2\r\n" 163 | b"+nested_set\r\n" 164 | b"~2\r\n" 165 | b"+element1\r\n" 166 | b"+element2\r\n" 167 | ) 168 | for i in range(10000): 169 | reader.feed(data) 170 | res = reader.gets() 171 | assert { 172 | b"radius": 4.5, 173 | b"diameter": 9, 174 | b"nested_map": {b"key1": b"value1", b"key2": 42}, 175 | b"nested_array": [b"item1", b"item2"], 176 | b"nested_set": [b"element1", b"element2"], 177 | } == res 178 | 179 | def test_dict_with_unhashable_key(reader): 180 | reader.feed( 181 | b"%1\r\n" 182 | b"%1\r\n+key1\r\n+value1\r\n" 183 | b":9\r\n" 184 | ) 185 | with pytest.raises(TypeError): 186 | reader.gets() 187 | 188 | def test_vector(reader): 189 | reader.feed(b">4\r\n+pubsub\r\n+message\r\n+channel\r\n+message\r\n") 190 | result = reader.gets() 191 | assert isinstance(result, hiredis.PushNotification) 192 | assert [b"pubsub", b"message", b"channel", b"message"] == result 193 | 194 | def test_verbatim_string(reader): 195 | value = b"text" 196 | reader.feed(b"=8\r\ntxt:%s\r\n" % value) 197 | assert value == reader.gets() 198 | 199 | def test_status_string(reader): 200 | reader.feed(b"+ok\r\n") 201 | assert b"ok" == reader.gets() 202 | 203 | def test_empty_bulk_string(reader): 204 | reader.feed(b"$0\r\n\r\n") 205 | assert b"" == reader.gets() 206 | 207 | def test_bulk_string(reader): 208 | reader.feed(b"$5\r\nhello\r\n") 209 | assert b"hello" == reader.gets() 210 | 211 | def test_bulk_string_without_encoding(reader): 212 | snowman = b"\xe2\x98\x83" 213 | reader.feed(b"$3\r\n" + snowman + b"\r\n") 214 | assert snowman == reader.gets() 215 | 216 | def test_bulk_string_with_encoding(): 217 | snowman = b"\xe2\x98\x83" 218 | r= hiredis.Reader(encoding="utf-8") 219 | r.feed(b"$3\r\n" + snowman + b"\r\n") 220 | assert snowman.decode("utf-8") == r.gets() 221 | 222 | def test_decode_errors_defaults_to_strict(): 223 | r= hiredis.Reader(encoding="utf-8") 224 | r.feed(b"+\x80\r\n") 225 | with pytest.raises(UnicodeDecodeError): 226 | r.gets() 227 | 228 | def test_decode_error_with_ignore_errors(): 229 | r= hiredis.Reader(encoding="utf-8", errors="ignore") 230 | r.feed(b"+\x80value\r\n") 231 | assert "value" == r.gets() 232 | 233 | def test_decode_error_with_surrogateescape_errors(): 234 | r= hiredis.Reader(encoding="utf-8", errors="surrogateescape") 235 | r.feed(b"+\x80value\r\n") 236 | assert "\udc80value" == r.gets() 237 | 238 | def test_invalid_encoding(): 239 | with pytest.raises(LookupError): 240 | hiredis.Reader(encoding="unknown") 241 | 242 | def test_should_decode_false_flag_prevents_decoding(): 243 | snowman = b"\xe2\x98\x83" 244 | r = hiredis.Reader(encoding="utf-8") 245 | r.feed(b"$3\r\n" + snowman + b"\r\n") 246 | r.feed(b"$3\r\n" + snowman + b"\r\n") 247 | assert snowman == r.gets(False) 248 | assert snowman.decode() == r.gets() 249 | 250 | def test_should_decode_true_flag_decodes_as_normal(): 251 | snowman = b"\xe2\x98\x83" 252 | r= hiredis.Reader(encoding="utf-8") 253 | r.feed(b"$3\r\n" + snowman + b"\r\n") 254 | assert snowman.decode() == r.gets(True) 255 | 256 | def test_set_encoding_with_different_encoding(): 257 | snowman_utf8 = b"\xe2\x98\x83" 258 | snowman_utf16 = b"\xff\xfe\x03&" 259 | r= hiredis.Reader(encoding="utf-8") 260 | r.feed(b"$3\r\n" + snowman_utf8 + b"\r\n") 261 | r.feed(b"$4\r\n" + snowman_utf16 + b"\r\n") 262 | assert snowman_utf8.decode() == r.gets() 263 | r.set_encoding(encoding="utf-16", errors="strict") 264 | assert snowman_utf16.decode('utf-16') == r.gets() 265 | 266 | def test_set_encoding_to_not_decode(): 267 | snowman = b"\xe2\x98\x83" 268 | r= hiredis.Reader(encoding="utf-8") 269 | r.feed(b"$3\r\n" + snowman + b"\r\n") 270 | r.feed(b"$3\r\n" + snowman + b"\r\n") 271 | assert snowman.decode() == r.gets() 272 | r.set_encoding(encoding=None, errors=None) 273 | assert snowman == r.gets() 274 | 275 | def test_set_encoding_invalid_encoding(): 276 | r= hiredis.Reader(encoding="utf-8") 277 | with pytest.raises(LookupError): 278 | r.set_encoding("unknown") 279 | 280 | def test_set_encoding_invalid_error_handler(): 281 | r = hiredis.Reader(encoding="utf-8") 282 | with pytest.raises(LookupError): 283 | r.set_encoding(encoding="utf-8", errors="unknown") 284 | 285 | def test_null_multi_bulk(reader): 286 | reader.feed(b"*-1\r\n") 287 | assert reader.gets() is None 288 | 289 | def test_empty_multi_bulk(reader): 290 | reader.feed(b"*0\r\n") 291 | assert reader.gets() == [] 292 | 293 | def test_multi_bulk(reader): 294 | reader.feed(b"*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n") 295 | assert [b"hello", b"world"] == reader.gets() 296 | 297 | def test_nested_multi_bulk(reader): 298 | reader.feed(b"*2\r\n*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n$1\r\n!\r\n") 299 | assert [[b"hello", b"world"], b"!"] == reader.gets() 300 | 301 | def test_nested_multi_bulk_depth(reader): 302 | reader.feed(b"*1\r\n*1\r\n*1\r\n*1\r\n$1\r\n!\r\n") 303 | assert [[[[b"!"]]]] == reader.gets() 304 | 305 | def test_subclassable(reader): 306 | 307 | class TestReader(hiredis.Reader): 308 | pass 309 | 310 | reader = TestReader() 311 | reader.feed(b"+ok\r\n") 312 | assert b"ok" == reader.gets() 313 | 314 | def test_invalid_offset(reader): 315 | data = b"+ok\r\n" 316 | with pytest.raises(ValueError): 317 | reader.feed(data, 6) 318 | 319 | def test_invalid_length(reader): 320 | data = b"+ok\r\n" 321 | with pytest.raises(ValueError): 322 | reader.feed(data, 0, 6) 323 | 324 | def test_ok_offset(reader): 325 | data = b"blah+ok\r\n" 326 | reader.feed(data, 4) 327 | assert b"ok" == reader.gets() 328 | 329 | def test_ok_length(reader): 330 | data = b"blah+ok\r\n" 331 | reader.feed(data, 4, len(data)-4) 332 | assert b"ok" == reader.gets() 333 | 334 | def test_feed_bytearray(reader): 335 | reader.feed(bytearray(b"+ok\r\n")) 336 | assert b"ok" == reader.gets() 337 | 338 | def test_maxbuf(reader): 339 | defaultmaxbuf = reader.getmaxbuf() 340 | reader.setmaxbuf(0) 341 | assert 0 == reader.getmaxbuf() 342 | reader.setmaxbuf(10000) 343 | assert 10000 == reader.getmaxbuf() 344 | reader.setmaxbuf(None) 345 | assert defaultmaxbuf == reader.getmaxbuf() 346 | with pytest.raises(ValueError): 347 | reader.setmaxbuf(-4) 348 | 349 | def test_len(reader): 350 | assert reader.len() == 0 351 | data = b"+ok\r\n" 352 | reader.feed(data) 353 | assert reader.len() == len(data) 354 | 355 | # hiredis reallocates and removes unused buffer once 356 | # there is at least 1K of not used data. 357 | calls = int((1024 / len(data))) + 1 358 | for i in range(calls): 359 | reader.feed(data) 360 | reader.gets() 361 | 362 | assert reader.len() == 5 363 | 364 | def test_reader_has_data(reader): 365 | assert reader.has_data() is False 366 | data = b"+ok\r\n" 367 | reader.feed(data) 368 | assert reader.has_data() 369 | reader.gets() 370 | assert reader.has_data() is False 371 | 372 | def test_custom_not_enough_data(): 373 | r = hiredis.Reader(notEnoughData=Ellipsis) 374 | assert r.gets() == Ellipsis 375 | -------------------------------------------------------------------------------- /src/reader.c: -------------------------------------------------------------------------------- 1 | #include "reader.h" 2 | 3 | #include 4 | #include 5 | 6 | static void Reader_dealloc(hiredis_ReaderObject *self); 7 | static int Reader_traverse(hiredis_ReaderObject *self, visitproc visit, void *arg); 8 | static int Reader_init(hiredis_ReaderObject *self, PyObject *args, PyObject *kwds); 9 | static PyObject *Reader_new(PyTypeObject *type, PyObject *args, PyObject *kwds); 10 | static PyObject *Reader_feed(hiredis_ReaderObject *self, PyObject *args); 11 | static PyObject *Reader_gets(hiredis_ReaderObject *self, PyObject *args); 12 | static PyObject *Reader_setmaxbuf(hiredis_ReaderObject *self, PyObject *arg); 13 | static PyObject *Reader_getmaxbuf(hiredis_ReaderObject *self); 14 | static PyObject *Reader_len(hiredis_ReaderObject *self); 15 | static PyObject *Reader_has_data(hiredis_ReaderObject *self); 16 | static PyObject *Reader_set_encoding(hiredis_ReaderObject *self, PyObject *args, PyObject *kwds); 17 | 18 | static int PushNotificationType_init(PushNotificationObject *self, PyObject *args, PyObject *kwds); 19 | /* Create a new instance of PushNotificationType with preallocated number of elements */ 20 | static PyObject* PushNotificationType_New(Py_ssize_t size); 21 | 22 | static PyMethodDef hiredis_ReaderMethods[] = { 23 | {"feed", (PyCFunction)Reader_feed, METH_VARARGS, NULL }, 24 | {"gets", (PyCFunction)Reader_gets, METH_VARARGS, NULL }, 25 | {"setmaxbuf", (PyCFunction)Reader_setmaxbuf, METH_O, NULL }, 26 | {"getmaxbuf", (PyCFunction)Reader_getmaxbuf, METH_NOARGS, NULL }, 27 | {"len", (PyCFunction)Reader_len, METH_NOARGS, NULL }, 28 | {"has_data", (PyCFunction)Reader_has_data, METH_NOARGS, NULL }, 29 | {"set_encoding", (PyCFunction)Reader_set_encoding, METH_VARARGS | METH_KEYWORDS, NULL }, 30 | { NULL } /* Sentinel */ 31 | }; 32 | 33 | PyTypeObject hiredis_ReaderType = { 34 | PyVarObject_HEAD_INIT(NULL, 0) 35 | MOD_HIREDIS ".Reader", /*tp_name*/ 36 | sizeof(hiredis_ReaderObject), /*tp_basicsize*/ 37 | 0, /*tp_itemsize*/ 38 | (destructor)Reader_dealloc, /*tp_dealloc*/ 39 | 0, /*tp_print*/ 40 | 0, /*tp_getattr*/ 41 | 0, /*tp_setattr*/ 42 | 0, /*tp_compare*/ 43 | 0, /*tp_repr*/ 44 | 0, /*tp_as_number*/ 45 | 0, /*tp_as_sequence*/ 46 | 0, /*tp_as_mapping*/ 47 | 0, /*tp_hash */ 48 | 0, /*tp_call*/ 49 | 0, /*tp_str*/ 50 | 0, /*tp_getattro*/ 51 | 0, /*tp_setattro*/ 52 | 0, /*tp_as_buffer*/ 53 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 54 | "Hiredis protocol reader", /*tp_doc */ 55 | (traverseproc)Reader_traverse,/*tp_traverse */ 56 | 0, /*tp_clear */ 57 | 0, /*tp_richcompare */ 58 | 0, /*tp_weaklistoffset */ 59 | 0, /*tp_iter */ 60 | 0, /*tp_iternext */ 61 | hiredis_ReaderMethods, /*tp_methods */ 62 | 0, /*tp_members */ 63 | 0, /*tp_getset */ 64 | 0, /*tp_base */ 65 | 0, /*tp_dict */ 66 | 0, /*tp_descr_get */ 67 | 0, /*tp_descr_set */ 68 | 0, /*tp_dictoffset */ 69 | (initproc)Reader_init, /*tp_init */ 70 | 0, /*tp_alloc */ 71 | Reader_new, /*tp_new */ 72 | }; 73 | 74 | PyTypeObject PushNotificationType = { 75 | PyVarObject_HEAD_INIT(NULL, 0) 76 | .tp_name = MOD_HIREDIS ".PushNotification", 77 | .tp_basicsize = sizeof(PushNotificationObject), 78 | .tp_itemsize = 0, 79 | .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, 80 | .tp_doc = "Redis PUSH notification type", 81 | .tp_init = (initproc) PushNotificationType_init, 82 | }; 83 | 84 | static void *tryParentize(const redisReadTask *task, PyObject *obj) { 85 | PyObject *parent; 86 | if (task && task->parent) { 87 | parent = (PyObject*)task->parent->obj; 88 | switch (task->parent->type) { 89 | case REDIS_REPLY_MAP: 90 | if (task->idx % 2 == 0) { 91 | /* Set a temporary item to save the object as a key. */ 92 | int res = PyDict_SetItem(parent, obj, Py_None); 93 | Py_DECREF(obj); 94 | 95 | if (res == -1) { 96 | return NULL; 97 | } 98 | } else { 99 | /* Pop the temporary item and set proper key and value. */ 100 | PyObject *last_item = PyObject_CallMethod(parent, "popitem", NULL); 101 | PyObject *last_key = PyTuple_GetItem(last_item, 0); 102 | PyDict_SetItem(parent, last_key, obj); 103 | Py_DECREF(last_item); 104 | Py_DECREF(obj); 105 | } 106 | break; 107 | default: 108 | assert(PyList_Check(parent)); 109 | PyList_SET_ITEM(parent, task->idx, obj); 110 | } 111 | } 112 | return obj; 113 | } 114 | 115 | static PyObject *createDecodedString(hiredis_ReaderObject *self, const char *str, size_t len) { 116 | PyObject *obj; 117 | 118 | if (self->encoding == NULL || !self->shouldDecode) { 119 | obj = PyBytes_FromStringAndSize(str, len); 120 | } else { 121 | obj = PyUnicode_Decode(str, len, self->encoding, self->errors); 122 | if (obj == NULL) { 123 | /* Store error when this is the first. */ 124 | if (self->error.ptype == NULL) 125 | PyErr_Fetch(&(self->error.ptype), &(self->error.pvalue), 126 | &(self->error.ptraceback)); 127 | 128 | /* Return Py_None as placeholder to let the error bubble up and 129 | * be used when a full reply in Reader#gets(). */ 130 | obj = Py_None; 131 | Py_INCREF(obj); 132 | PyErr_Clear(); 133 | } 134 | } 135 | 136 | assert(obj != NULL); 137 | return obj; 138 | } 139 | 140 | static void *createError(PyObject *errorCallable, char *errstr, size_t len) { 141 | PyObject *obj, *errmsg; 142 | 143 | errmsg = PyUnicode_DecodeUTF8(errstr, len, "replace"); 144 | assert(errmsg != NULL); /* TODO: properly handle OOM etc */ 145 | 146 | obj = PyObject_CallFunctionObjArgs(errorCallable, errmsg, NULL); 147 | Py_DECREF(errmsg); 148 | /* obj can be NULL if custom error class raised another exception */ 149 | 150 | return obj; 151 | } 152 | 153 | static void *createStringObject(const redisReadTask *task, char *str, size_t len) { 154 | hiredis_ReaderObject *self = (hiredis_ReaderObject*)task->privdata; 155 | PyObject *obj; 156 | 157 | if (task->type == REDIS_REPLY_ERROR) { 158 | obj = createError(self->replyErrorClass, str, len); 159 | if (obj == NULL) { 160 | if (self->error.ptype == NULL) 161 | PyErr_Fetch(&(self->error.ptype), &(self->error.pvalue), 162 | &(self->error.ptraceback)); 163 | obj = Py_None; 164 | Py_INCREF(obj); 165 | } 166 | } else { 167 | if (task->type == REDIS_REPLY_VERB) { 168 | /* Skip 4 bytes of verbatim type header. */ 169 | memmove(str, str+4, len); 170 | len -= 4; 171 | } 172 | obj = createDecodedString(self, str, len); 173 | } 174 | return tryParentize(task, obj); 175 | } 176 | 177 | static void *createArrayObject(const redisReadTask *task, size_t elements) { 178 | PyObject *obj; 179 | switch (task->type) { 180 | case REDIS_REPLY_MAP: 181 | obj = PyDict_New(); 182 | break; 183 | case REDIS_REPLY_PUSH: 184 | obj = PushNotificationType_New(elements); 185 | break; 186 | default: 187 | obj = PyList_New(elements); 188 | } 189 | return tryParentize(task, obj); 190 | } 191 | 192 | static void *createIntegerObject(const redisReadTask *task, long long value) { 193 | PyObject *obj; 194 | obj = PyLong_FromLongLong(value); 195 | return tryParentize(task, obj); 196 | } 197 | 198 | static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t le) { 199 | PyObject *obj; 200 | obj = PyFloat_FromDouble(value); 201 | return tryParentize(task, obj); 202 | } 203 | 204 | static void *createNilObject(const redisReadTask *task) { 205 | PyObject *obj = Py_None; 206 | Py_INCREF(obj); 207 | return tryParentize(task, obj); 208 | } 209 | 210 | static void *createBoolObject(const redisReadTask *task, int bval) { 211 | PyObject *obj; 212 | obj = PyBool_FromLong((long)bval); 213 | return tryParentize(task, obj); 214 | } 215 | 216 | static void freeObject(void *obj) { 217 | Py_XDECREF(obj); 218 | } 219 | 220 | static int PushNotificationType_init(PushNotificationObject *self, PyObject *args, PyObject *kwds) { 221 | return PyList_Type.tp_init((PyObject *)self, args, kwds); 222 | } 223 | 224 | static PyObject* PushNotificationType_New(Py_ssize_t size) { 225 | /* Check for negative size */ 226 | if (size < 0) { 227 | PyErr_SetString(PyExc_SystemError, "negative list size"); 228 | return NULL; 229 | } 230 | 231 | /* Check for potential overflow */ 232 | if ((size_t)size > PY_SSIZE_T_MAX / sizeof(PyObject*)) { 233 | return PyErr_NoMemory(); 234 | } 235 | 236 | #ifdef PYPY_VERSION 237 | PyObject* obj = PyObject_CallObject((PyObject *) &PushNotificationType, NULL); 238 | #else 239 | PyObject* obj = PyType_GenericNew(&PushNotificationType, NULL, NULL); 240 | #endif 241 | if (obj == NULL) { 242 | return NULL; 243 | } 244 | 245 | int res = PyList_SetSlice(obj, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PyList_New(size)); 246 | 247 | if (res == -1) { 248 | Py_DECREF(obj); 249 | return NULL; 250 | } 251 | 252 | return obj; 253 | } 254 | 255 | redisReplyObjectFunctions hiredis_ObjectFunctions = { 256 | createStringObject, // void *(*createString)(const redisReadTask*, char*, size_t); 257 | createArrayObject, // void *(*createArray)(const redisReadTask*, size_t); 258 | createIntegerObject, // void *(*createInteger)(const redisReadTask*, long long); 259 | createDoubleObject, // void *(*createDoubleObject)(const redisReadTask*, double, char*, size_t); 260 | createNilObject, // void *(*createNil)(const redisReadTask*); 261 | createBoolObject, // void *(*createBoolObject)(const redisReadTask*, int); 262 | freeObject // void (*freeObject)(void*); 263 | }; 264 | 265 | static void Reader_dealloc(hiredis_ReaderObject *self) { 266 | PyObject_GC_UnTrack(self); 267 | // we don't need to free self->encoding as the buffer is managed by Python 268 | // https://docs.python.org/3/c-api/arg.html#strings-and-buffers 269 | redisReaderFree(self->reader); 270 | Py_CLEAR(self->protocolErrorClass); 271 | Py_CLEAR(self->replyErrorClass); 272 | Py_CLEAR(self->notEnoughDataObject); 273 | 274 | ((PyObject *)self)->ob_type->tp_free((PyObject*)self); 275 | } 276 | 277 | static int Reader_traverse(hiredis_ReaderObject *self, visitproc visit, void *arg) { 278 | Py_VISIT(self->protocolErrorClass); 279 | Py_VISIT(self->replyErrorClass); 280 | Py_VISIT(self->notEnoughDataObject); 281 | return 0; 282 | } 283 | 284 | static int _Reader_set_exception(PyObject **target, PyObject *value) { 285 | int callable; 286 | callable = PyCallable_Check(value); 287 | 288 | if (callable == 0) { 289 | PyErr_SetString(PyExc_TypeError, "Expected a callable"); 290 | return 0; 291 | } 292 | 293 | Py_DECREF(*target); 294 | *target = value; 295 | Py_INCREF(*target); 296 | return 1; 297 | } 298 | 299 | static int _Reader_set_encoding(hiredis_ReaderObject *self, char *encoding, char *errors) { 300 | PyObject *codecs, *result; 301 | 302 | if (encoding) { // validate that the encoding exists, raises LookupError if not 303 | codecs = PyImport_ImportModule("codecs"); 304 | if (!codecs) 305 | return -1; 306 | result = PyObject_CallMethod(codecs, "lookup", "s", encoding); 307 | Py_DECREF(codecs); 308 | if (!result) 309 | return -1; 310 | Py_DECREF(result); 311 | self->encoding = encoding; 312 | } else { 313 | self->encoding = NULL; 314 | } 315 | 316 | if (errors) { // validate that the error handler exists, raises LookupError if not 317 | codecs = PyImport_ImportModule("codecs"); 318 | if (!codecs) 319 | return -1; 320 | result = PyObject_CallMethod(codecs, "lookup_error", "s", errors); 321 | Py_DECREF(codecs); 322 | if (!result) 323 | return -1; 324 | Py_DECREF(result); 325 | self->errors = errors; 326 | } else { 327 | self->errors = "strict"; 328 | } 329 | 330 | return 0; 331 | } 332 | 333 | static int Reader_init(hiredis_ReaderObject *self, PyObject *args, PyObject *kwds) { 334 | static char *kwlist[] = { "protocolError", "replyError", "encoding", "errors", "notEnoughData", NULL }; 335 | PyObject *protocolErrorClass = NULL; 336 | PyObject *replyErrorClass = NULL; 337 | PyObject *notEnoughData = NULL; 338 | char *encoding = NULL; 339 | char *errors = NULL; 340 | 341 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOzzO", kwlist, 342 | &protocolErrorClass, &replyErrorClass, &encoding, &errors, ¬EnoughData)) 343 | return -1; 344 | 345 | if (protocolErrorClass) 346 | if (!_Reader_set_exception(&self->protocolErrorClass, protocolErrorClass)) 347 | return -1; 348 | 349 | if (replyErrorClass) 350 | if (!_Reader_set_exception(&self->replyErrorClass, replyErrorClass)) 351 | return -1; 352 | 353 | if (notEnoughData) { 354 | Py_DECREF(self->notEnoughDataObject); 355 | self->notEnoughDataObject = notEnoughData; 356 | 357 | Py_INCREF(self->notEnoughDataObject); 358 | } 359 | 360 | return _Reader_set_encoding(self, encoding, errors); 361 | } 362 | 363 | static PyObject *Reader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { 364 | hiredis_ReaderObject *self; 365 | self = (hiredis_ReaderObject*)type->tp_alloc(type, 0); 366 | if (self != NULL) { 367 | self->reader = redisReaderCreateWithFunctions(NULL); 368 | self->reader->fn = &hiredis_ObjectFunctions; 369 | self->reader->privdata = self; 370 | 371 | self->encoding = NULL; 372 | self->errors = "strict"; // default to "strict" to mimic Python 373 | self->notEnoughDataObject = Py_False; 374 | self->shouldDecode = 1; 375 | self->protocolErrorClass = HIREDIS_STATE->HiErr_ProtocolError; 376 | self->replyErrorClass = HIREDIS_STATE->HiErr_ReplyError; 377 | Py_INCREF(self->protocolErrorClass); 378 | Py_INCREF(self->replyErrorClass); 379 | Py_INCREF(self->notEnoughDataObject); 380 | 381 | self->error.ptype = NULL; 382 | self->error.pvalue = NULL; 383 | self->error.ptraceback = NULL; 384 | } 385 | return (PyObject*)self; 386 | } 387 | 388 | static PyObject *Reader_feed(hiredis_ReaderObject *self, PyObject *args) { 389 | Py_buffer buf; 390 | Py_ssize_t off = 0; 391 | Py_ssize_t len = -1; 392 | 393 | if (!PyArg_ParseTuple(args, "s*|nn", &buf, &off, &len)) { 394 | return NULL; 395 | } 396 | 397 | if (len == -1) { 398 | len = buf.len - off; 399 | } 400 | 401 | if (off < 0 || len < 0) { 402 | PyErr_SetString(PyExc_ValueError, "negative input"); 403 | goto error; 404 | } 405 | 406 | if ((off + len) > buf.len) { 407 | PyErr_SetString(PyExc_ValueError, "input is larger than buffer size"); 408 | goto error; 409 | } 410 | 411 | redisReaderFeed(self->reader, (char *)buf.buf + off, len); 412 | PyBuffer_Release(&buf); 413 | Py_RETURN_NONE; 414 | 415 | error: 416 | PyBuffer_Release(&buf); 417 | return NULL; 418 | } 419 | 420 | static PyObject *Reader_gets(hiredis_ReaderObject *self, PyObject *args) { 421 | PyObject *obj; 422 | 423 | self->shouldDecode = 1; 424 | if (!PyArg_ParseTuple(args, "|i", &self->shouldDecode)) { 425 | return NULL; 426 | } 427 | 428 | if (redisReaderGetReply(self->reader, (void**)&obj) == REDIS_ERR) { 429 | PyObject *err = NULL; 430 | char *errstr = NULL; 431 | 432 | // Checking if there is no error during the call to redisReaderGetReply 433 | // to avoid getting a SystemError. 434 | if (PyErr_Occurred() == NULL) { 435 | errstr = redisReaderGetError(self->reader); 436 | /* protocolErrorClass might be a callable. call it, then use it's type */ 437 | err = createError(self->protocolErrorClass, errstr, strlen(errstr)); 438 | } 439 | 440 | if (err != NULL) { 441 | obj = PyObject_Type(err); 442 | PyErr_SetString(obj, errstr); 443 | Py_DECREF(obj); 444 | Py_DECREF(err); 445 | } 446 | return NULL; 447 | } 448 | 449 | if (obj == NULL) { 450 | Py_INCREF(self->notEnoughDataObject); 451 | return self->notEnoughDataObject; 452 | } else { 453 | /* Restore error when there is one. */ 454 | if (self->error.ptype != NULL) { 455 | Py_DECREF(obj); 456 | PyErr_Restore(self->error.ptype, self->error.pvalue, 457 | self->error.ptraceback); 458 | self->error.ptype = NULL; 459 | self->error.pvalue = NULL; 460 | self->error.ptraceback = NULL; 461 | return NULL; 462 | } 463 | return obj; 464 | } 465 | } 466 | 467 | static PyObject *Reader_setmaxbuf(hiredis_ReaderObject *self, PyObject *arg) { 468 | long maxbuf; 469 | 470 | if (arg == Py_None) 471 | maxbuf = REDIS_READER_MAX_BUF; 472 | else { 473 | maxbuf = PyLong_AsLong(arg); 474 | if (maxbuf < 0) { 475 | if (!PyErr_Occurred()) 476 | PyErr_SetString(PyExc_ValueError, 477 | "maxbuf value out of range"); 478 | return NULL; 479 | } 480 | } 481 | self->reader->maxbuf = maxbuf; 482 | 483 | Py_INCREF(Py_None); 484 | return Py_None; 485 | } 486 | 487 | static PyObject *Reader_getmaxbuf(hiredis_ReaderObject *self) { 488 | return PyLong_FromSize_t(self->reader->maxbuf); 489 | } 490 | 491 | static PyObject *Reader_len(hiredis_ReaderObject *self) { 492 | return PyLong_FromSize_t(self->reader->len); 493 | } 494 | 495 | static PyObject *Reader_has_data(hiredis_ReaderObject *self) { 496 | if(self->reader->pos < self->reader->len) 497 | Py_RETURN_TRUE; 498 | Py_RETURN_FALSE; 499 | } 500 | 501 | static PyObject *Reader_set_encoding(hiredis_ReaderObject *self, PyObject *args, PyObject *kwds) { 502 | static char *kwlist[] = { "encoding", "errors", NULL }; 503 | char *encoding = NULL; 504 | char *errors = NULL; 505 | 506 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "|zz", kwlist, &encoding, &errors)) 507 | return NULL; 508 | 509 | if(_Reader_set_encoding(self, encoding, errors) == -1) 510 | return NULL; 511 | 512 | Py_RETURN_NONE; 513 | 514 | } 515 | --------------------------------------------------------------------------------