├── .gitignore ├── .travis.mk ├── .travis.yml ├── LICENSE ├── Makefile ├── NEWS ├── README.rst ├── commands.c ├── compat.h ├── compat.mk ├── debian ├── .gitignore ├── changelog.in ├── compat ├── control ├── copyright ├── docs ├── libomcache-dev.install ├── libomcache0.install └── rules ├── dist.c ├── doc ├── .gitignore ├── Doxyfile ├── Makefile ├── conf.py └── omcache_cdef.rst ├── md5.c ├── memcached_protocol_binary.h ├── omcache.c ├── omcache.h ├── omcache.py ├── omcache.spec ├── omcache_cdef.h ├── omcache_libmemcached.h ├── omcache_priv.h ├── omcache_pylibmc.py ├── pylintrc ├── symbol.map ├── tests ├── Makefile ├── python │ ├── __init__.py │ └── test_omcache.py ├── test_commands.c ├── test_failures.c ├── test_libmcd_compat.c ├── test_misc.c ├── test_omcache.c ├── test_omcache.h └── test_servers.c ├── util.c └── version.mk /.gitignore: -------------------------------------------------------------------------------- 1 | .coverity-email 2 | .coverity-token 3 | *.gcda 4 | *.gcno 5 | *.gcov 6 | *.o 7 | *.pyc 8 | *~ 9 | __pycache__/ 10 | libomcache.a 11 | libomcache.so* 12 | tests/test_omcache 13 | -------------------------------------------------------------------------------- /.travis.mk: -------------------------------------------------------------------------------- 1 | all: $(shell uname -s) 2 | 3 | Linux: 4 | sudo apt-get update 5 | sudo apt-get install libasyncns-dev check memcached valgrind 6 | pip install cffi pytest pylint 7 | $(MAKE) all 8 | # We don't need to run C unit tests under valgrind for every python 9 | # version, run them only when we're testing Python 2.6 as we'll skip 10 | # the pylint checks when running it as pylint no longer supports 2.6 11 | (python -V 2>&1 | grep -qvF 'Python 2.6') || $(MAKE) check-valgrind 12 | (python -V 2>&1 | grep -qF 'Python 2.6') || $(MAKE) check-pylint 13 | $(MAKE) check-python 14 | 15 | Darwin: 16 | brew update 17 | brew install check || : 18 | $(MAKE) WITHOUT_ASYNCNS=1 all 19 | $(MAKE) WITHOUT_ASYNCNS=1 check 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | script: make -f .travis.mk 2 | 3 | matrix: 4 | include: 5 | - os: osx 6 | language: c 7 | compiler: clang 8 | - os: linux 9 | language: python 10 | python: "2.6" 11 | compiler: gcc 12 | env: CC=gcc 13 | - os: linux 14 | language: python 15 | python: "2.7" 16 | compiler: clang 17 | env: CC=clang 18 | - os: linux 19 | language: python 20 | python: "3.3" 21 | compiler: gcc 22 | env: CC=gcc 23 | - os: linux 24 | language: python 25 | python: "3.4" 26 | compiler: clang 27 | env: CC=clang 28 | - os: linux 29 | language: python 30 | python: "pypy" 31 | compiler: clang 32 | env: CC=clang 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include compat.mk 2 | include version.mk 3 | 4 | PREFIX ?= /usr/local 5 | LIBDIR ?= $(PREFIX)/lib 6 | INCLUDEDIR ?= $(PREFIX)/include 7 | PYTHON ?= python 8 | 9 | STLIB_A = libomcache.a 10 | SHLIB_SO = libomcache.$(SO_EXT) 11 | SHLIB_V = $(SHLIB_SO).0 12 | OBJ = omcache.o commands.o dist.o md5.o util.o 13 | 14 | 15 | all: $(SHLIB_SO) $(STLIB_A) 16 | 17 | $(STLIB_A): $(OBJ) 18 | ar rc $@ $^ 19 | ranlib $@ 20 | 21 | $(SHLIB_SO): $(SHLIB_V) 22 | ln -fs $(SHLIB_V) $(SHLIB_SO) 23 | 24 | $(SHLIB_V): $(OBJ) symbol.map 25 | $(CC) $(LDFLAGS) $(SO_FLAGS) $(filter-out symbol.map,$^) -o $@ $(WITH_LIBS) 26 | 27 | install: $(SHLIB_SO) 28 | mkdir -p $(DESTDIR)$(LIBDIR) $(DESTDIR)$(INCLUDEDIR) 29 | cp -a $(SHLIB_SO) $(SHLIB_V) $(DESTDIR)$(LIBDIR) 30 | cp -a omcache.h omcache_cdef.h omcache_libmemcached.h $(DESTDIR)$(INCLUDEDIR) 31 | ifneq ($(PYTHONDIRS),) 32 | for pydir in $(PYTHONDIRS); do \ 33 | mkdir -p $(DESTDIR)$$pydir; \ 34 | cp -a omcache.py omcache_pylibmc.py omcache_cdef.h \ 35 | $(DESTDIR)$$pydir; \ 36 | done 37 | endif 38 | 39 | rpm: 40 | git archive --output=omcache-rpm-src.tar.gz --prefix=omcache/ HEAD 41 | rpmbuild -bb omcache.spec \ 42 | --define '_sourcedir $(shell pwd)' \ 43 | --define 'major_version $(short_ver)' \ 44 | --define 'minor_version $(subst -,.,$(subst $(short_ver)-,,$(long_ver)))' 45 | $(RM) omcache-rpm-src.tar.gz 46 | 47 | deb: 48 | cp debian/changelog.in debian/changelog 49 | dch -v $(long_ver) "Automatically built package" 50 | dpkg-buildpackage -uc -us 51 | 52 | clean: 53 | $(RM) $(STLIB_A) $(SHLIB_V) $(SHLIB_SO) $(OBJ) 54 | $(MAKE) -C tests clean 55 | 56 | check: 57 | $(MAKE) -C tests check 58 | 59 | check-sanitizer: 60 | $(MAKE) clean 61 | $(MAKE) CFLAGS="$(CFLAGS) -fsanitize=address,undefined" \ 62 | LDFLAGS="$(LDFLAGS) -fsanitize=address,undefined" \ 63 | -C tests check 64 | 65 | check-valgrind: 66 | $(MAKE) clean 67 | $(MAKE) 68 | $(MAKE) -C tests check CHECKER="valgrind --leak-check=full" 69 | 70 | check-coverage: 71 | $(MAKE) clean 72 | $(MAKE) CFLAGS="$(CFLAGS) -fprofile-arcs -ftest-coverage" \ 73 | LDFLAGS="$(LDFLAGS) -fprofile-arcs -ftest-coverage" \ 74 | -C tests check 75 | gcov -rb $(OBJ:.o=.c) 76 | 77 | check-coverity: 78 | $(MAKE) clean 79 | $(RM) -r cov-int omcache-cov-int.tar.gz 80 | cov-build --dir cov-int $(MAKE) 81 | tar zcvf omcache-cov-int.tar.gz cov-int 82 | curl --verbose --form 'token=<.coverity-token' \ 83 | --form 'email=<.coverity-email' \ 84 | --form 'file=@omcache-cov-int.tar.gz' \ 85 | --form 'version=$(long_ver)' \ 86 | --form 'description=$(short_ver)' \ 87 | 'https://scan.coverity.com/builds?project=saaros%2Fomcache' 88 | $(RM) -r cov-int omcache-cov-int.tar.gz 89 | 90 | check-pylint: 91 | $(PYTHON) -m pylint --rcfile pylintrc *.py 92 | PYTHONPATH=. $(PYTHON) -m pylint --rcfile pylintrc tests/python 93 | 94 | check-python: 95 | LD_LIBRARY_PATH=. PYTHONPATH=. $(PYTHON) -m pytest -vv tests/python 96 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | OMcache 0.X (unreleased) 2 | ======================== 3 | 4 | * Compile-time configurable timeouts for libmemcached compat wrapper 5 | 6 | OMcache 0.3.0 (2015-02-15) 7 | ========================== 8 | 9 | * Support for OS X and Solaris 10 | * Support for Python 2.6, 2.7 and 3.3+ and PyPy 11 | * Unit tests for Python and related fixes for the Python bindings 12 | * Automatic tests on travis-ci.org for all supported Python versions 13 | * C API reference documentation generation 14 | * Various bug fixes 15 | 16 | OMcache 0.2.0 (2014-12-21) 17 | ========================== 18 | 19 | * Support for asynchronous name resolution using libasyncns 20 | * Added support for append, prepend, touch and gat commands 21 | * Improved and fixed timeout handling 22 | * Improved handling of "multi" commands 23 | * Better handling of error cases 24 | * Rewritten Python bindings for async support using gevent or other 25 | mechanisms 26 | * Various bug fixes 27 | 28 | OMcache 0.1.0 (2014-11-13) 29 | ========================== 30 | 31 | * Initial (unstable) release 32 | * Only tested on recent Linux distributions on x86-64 33 | * Support for most (but not all) memcached commands 34 | * C library with Python bindings implemented in CFFI 35 | ' Fedora and Debian packaging 36 | * Non-blocking API 37 | * Stable (without promises before 1.0 release) API and ABI 38 | * Support for consistent key distribution to multiple servers 39 | * Compatibility with libmemcached and pylibmc in a number of different 40 | configurations 41 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | OMcache |BuildStatus|_ |CoverityStatus|_ |DocBadge|_ 2 | ==================================================== 3 | 4 | .. |CoverityStatus| image:: https://scan.coverity.com/projects/3408/badge.svg 5 | .. _CoverityStatus: https://scan.coverity.com/projects/3408/ 6 | .. |BuildStatus| image:: https://travis-ci.org/ohmu/omcache.png?branch=master 7 | .. _BuildStatus: https://travis-ci.org/ohmu/omcache 8 | .. |DocBadge| image:: https://readthedocs.org/projects/omcache/badge/?version=latest 9 | .. _DocBadge: http://omcache.readthedocs.org/en/latest/ 10 | 11 | OMcache is a C and Python library for accessing memcached servers. The 12 | goals of the OMcache project are a stable API and ABI and 'easy' integration 13 | into complex applications and systems. OMcache is meant to be used as a 14 | middleware layer to provide redundancy and consistency to a memcached server 15 | cluster. OMcache specifically tries to avoid ABI breakage and does not mask 16 | any signals or call blocking functions to help integrating it into other 17 | solutions. 18 | 19 | :: 20 | 21 | ,##, 22 | OMcache ;@@@@@; 23 | ;#, `@@' ;@@, 24 | ,,...., @@, `@@@@@` 25 | ,'#@@@@@@@@@#;` `#@@@#; .@@@@@@@` 26 | #@@@@@@@@@@@@@@@@, `#@@@#+++#@@@@@@@@@@. 27 | `@@' @@# `#@@@@@@@@@@@@#. 28 | ;@@@@, 29 | .#@@@@@@@ 30 | ``.;#@@@@@@@@@' .:'#@@#;;, 31 | '@@@@@@@@@@@@@ ,#@@@@@@@@@@@@@@@@' 32 | `@@@' `@@@##@@@@@@@@@@@',` `#@@;, 33 | @@@@@@@@+. `#@@# 34 | ,##;, :@@@@@; '#@@@@@@@@@` 35 | ,@@#;, `#@@@@@+ `@@@@@@@@@@@# 36 | `@@@@@@@@@@@@@@@@@@ #@@@@' 37 | `#@@@@@@@@@@#' `@' 38 | 39 | 40 | Platform requirements 41 | ===================== 42 | 43 | OMcache has been developed and tested on x86-64 Linux, but it should work on 44 | other Linux architectures as well as Mac OS X on x86 and Solaris on Sparc 45 | without any changes. 46 | Other platforms will probably require some changes. 47 | 48 | OMcache can be built with recent versions of GCC and Clang. Clang and GCC 49 | versions prior to 4.9 will emit some `spurious warnings`_ about missing 50 | field initializers. Asynchronous name lookups require the libasyncns_ 51 | library; OMcache can be built without it by passing WITHOUT_ASYNCNS=1 52 | argument to make, but that will cause name lookups to become blocking 53 | operations. 54 | 55 | Unit tests are implemented using the Check_ unit testing framework. Check 56 | version 0.9.10 or newer is recommended, earlier versions can be used but 57 | their log output is limited. 58 | 59 | The Python module requires CFFI_ 0.6+ and supports CPython_ 2.6, 2.7 and 60 | 3.3+ and PyPy_ 2.2+. 61 | 62 | .. _`spurious warnings`: https://github.com/ohmu/omcache/issues/11 63 | .. _libasyncns: http://0pointer.de/lennart/projects/libasyncns/ 64 | .. _Check: http://check.sourceforge.net/ 65 | .. _CFFI: https://cffi.readthedocs.org/ 66 | .. _CPython: https://www.python.org/ 67 | .. _PyPy: http://pypy.org/ 68 | 69 | Compatibility 70 | ============= 71 | 72 | OMcache tries to be compatible with libmemcached_ where possible. During 73 | normal operations where all servers are available and no failovers have 74 | happened the two memcache clients should always select the same servers for 75 | keys. Libmemcached's failover mechanism has traditionally been poorly 76 | documented and its details have changed occasionally between the releases 77 | so 100% compatibility is not possible. 78 | 79 | OMcache supports configurable `key distribution algorithms`_, the default is 80 | the one used by libmemcached version 1.0.10 and newer, called 81 | ``omcache_dist_libmemcached_ketama`` in OMcache. Libmemcached versions 82 | prior to 1.0.10 had a bug in the implementation of the algorithm which 83 | caused it to use a mix of the normal ketama and weighted ketama, using 84 | Jenkins hashing for keys and md5 hashing for hosts. OMcache supports this 85 | algorithm as ``omcache_dist_libmemcached_ketama_pre1010``. 86 | 87 | OMcache also provides a thin API compatibility wrapper header which allows 88 | simple applications to be converted to use OMcache instead of libmemcached 89 | by including "omcache_libmemcached.h" instead of "libmemcached/memcached.h". 90 | Similarly there's a compatibility layer for pylibmc_ in omcache_pylibmc.py 91 | which provides a limited pylibmc-like API for OMcache. 92 | 93 | The functionality provided by these wrappers is limited and they are not 94 | supported, they're provided to make it easy to test OMcache in simple 95 | programs that use the libmemcached and pylibmc APIs. 96 | 97 | .. _`key distribution algorithms`: http://en.wikipedia.org/wiki/Consistent_hashing 98 | .. _libmemcached: http://libmemcached.org/ 99 | .. _pylibmc: http://sendapatch.se/projects/pylibmc/ 100 | 101 | License 102 | ======= 103 | 104 | OMcache is released under the Apache License, Version 2.0. 105 | 106 | For the exact license terms, see `LICENSE` and 107 | http://opensource.org/licenses/Apache-2.0 . 108 | 109 | Credits 110 | ======= 111 | 112 | OMcache was created by Oskari Saarenmaa and is maintained by 113 | Ohmu Ltd's hackers . 114 | 115 | F-Secure Corporation provided the infrastructure for testing OMcache in its 116 | initial development process and contributed to the pylibmc and libmemcached 117 | compatibility layers as well as ported pgmemcache (PostgreSQL memcached 118 | interface) to run on OMcache. 119 | 120 | MD5 implementation is based on Solar Designer's Public Domain / BSD licensed 121 | code from openwall.info. 122 | 123 | Contact 124 | ======= 125 | 126 | Bug reports and patches are very welcome, please post them as GitHub issues 127 | and pull requests at https://github.com/ohmu/omcache 128 | -------------------------------------------------------------------------------- /commands.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Commands for OMcache 3 | * 4 | * Copyright (c) 2013-2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "compat.h" 18 | // Note: commands.c only uses the "public" omcache api 19 | #include "omcache.h" 20 | #include "memcached_protocol_binary.h" 21 | 22 | #define QCMD(k) (timeout_msec ? k : k ## Q) 23 | 24 | 25 | int omcache_noop(omcache_t *mc, int server_index, int32_t timeout_msec) 26 | { 27 | omcache_req_t req[1] = {{ 28 | .server_index = server_index, 29 | .header = { 30 | .opcode = PROTOCOL_BINARY_CMD_NOOP, 31 | }, 32 | }}; 33 | return omcache_command_status(mc, req, timeout_msec); 34 | } 35 | 36 | int omcache_stat(omcache_t *mc, const char *command, 37 | omcache_value_t *values, size_t *value_count, 38 | int server_index, int32_t timeout_msec) 39 | { 40 | size_t key_len = command ? strlen(command) : 0; 41 | size_t req_count = 1; 42 | omcache_req_t req[1] = {{ 43 | .server_index = server_index, 44 | .header = { 45 | .opcode = PROTOCOL_BINARY_CMD_STAT, 46 | .keylen = htobe16(key_len), 47 | .bodylen = htobe32(key_len), 48 | }, 49 | .key = (const unsigned char *) command, 50 | }}; 51 | return omcache_command(mc, req, &req_count, values, value_count, timeout_msec); 52 | } 53 | 54 | int omcache_flush_all(omcache_t *mc, time_t expiration, int server_index, int32_t timeout_msec) 55 | { 56 | uint32_t body_exp = htobe32(expiration); 57 | omcache_req_t req[1] = {{ 58 | .server_index = server_index, 59 | .header = { 60 | .opcode = PROTOCOL_BINARY_CMD_FLUSH, 61 | .extlen = sizeof(body_exp), 62 | .bodylen = htobe32(sizeof(body_exp)), 63 | }, 64 | .extra = &body_exp, 65 | }}; 66 | return omcache_command_status(mc, req, timeout_msec); 67 | } 68 | 69 | struct protocol_binary_set_request_body_s { 70 | uint32_t flags; 71 | uint32_t expiration; 72 | } __attribute__((packed)); 73 | 74 | static int omc_set_cmd(omcache_t *mc, protocol_binary_command opcode, 75 | const unsigned char *key, size_t key_len, 76 | const unsigned char *value, size_t value_len, 77 | time_t expiration, uint32_t flags, 78 | uint64_t cas, int32_t timeout_msec) 79 | { 80 | struct protocol_binary_set_request_body_s extra = { 81 | .flags = htobe32(flags), 82 | .expiration = htobe32(expiration), 83 | }; 84 | size_t extra_len = sizeof(extra); 85 | if (opcode == PROTOCOL_BINARY_CMD_APPEND || opcode == PROTOCOL_BINARY_CMD_APPENDQ || 86 | opcode == PROTOCOL_BINARY_CMD_PREPEND || opcode == PROTOCOL_BINARY_CMD_PREPENDQ) 87 | extra_len = 0; 88 | omcache_req_t req[1] = {{ 89 | .server_index = -1, 90 | .header = { 91 | .opcode = opcode, 92 | .extlen = extra_len, 93 | .keylen = htobe16(key_len), 94 | .bodylen = htobe32(key_len + value_len + extra_len), 95 | .cas = htobe64(cas), 96 | }, 97 | .extra = &extra, 98 | .key = key, 99 | .data = value, 100 | }}; 101 | return omcache_command_status(mc, req, timeout_msec); 102 | } 103 | 104 | int omcache_set(omcache_t *mc, 105 | const unsigned char *key, size_t key_len, 106 | const unsigned char *value, size_t value_len, 107 | time_t expiration, uint32_t flags, 108 | uint64_t cas, int32_t timeout_msec) 109 | { 110 | return omc_set_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_SET), 111 | key, key_len, value, value_len, 112 | expiration, flags, cas, timeout_msec); 113 | } 114 | 115 | int omcache_add(omcache_t *mc, 116 | const unsigned char *key, size_t key_len, 117 | const unsigned char *value, size_t value_len, 118 | time_t expiration, uint32_t flags, 119 | int32_t timeout_msec) 120 | { 121 | return omc_set_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_ADD), 122 | key, key_len, value, value_len, 123 | expiration, flags, 0, timeout_msec); 124 | } 125 | 126 | int omcache_replace(omcache_t *mc, 127 | const unsigned char *key, size_t key_len, 128 | const unsigned char *value, size_t value_len, 129 | time_t expiration, uint32_t flags, 130 | int32_t timeout_msec) 131 | { 132 | return omc_set_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_REPLACE), 133 | key, key_len, value, value_len, 134 | expiration, flags, 0, timeout_msec); 135 | } 136 | 137 | int omcache_append(omcache_t *mc, 138 | const unsigned char *key, size_t key_len, 139 | const unsigned char *value, size_t value_len, 140 | uint64_t cas, int32_t timeout_msec) 141 | { 142 | return omc_set_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_APPEND), 143 | key, key_len, value, value_len, 144 | 0, 0, cas, timeout_msec); 145 | } 146 | 147 | int omcache_prepend(omcache_t *mc, 148 | const unsigned char *key, size_t key_len, 149 | const unsigned char *value, size_t value_len, 150 | uint64_t cas, int32_t timeout_msec) 151 | { 152 | return omc_set_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_PREPEND), 153 | key, key_len, value, value_len, 154 | 0, 0, cas, timeout_msec); 155 | } 156 | 157 | struct protocol_binary_delta_request_body_s { 158 | uint64_t delta; 159 | uint64_t initial; 160 | uint32_t expiration; 161 | } __attribute__((packed)); 162 | 163 | static int omc_ctr_cmd(omcache_t *mc, protocol_binary_command opcode, 164 | const unsigned char *key, size_t key_len, 165 | uint64_t delta, uint64_t initial, 166 | time_t expiration, uint64_t *valuep, 167 | int32_t timeout_msec) 168 | { 169 | struct protocol_binary_delta_request_body_s body = { 170 | .delta = htobe64(delta), 171 | .initial = htobe64(initial), 172 | .expiration = htobe32(expiration), 173 | }; 174 | omcache_req_t req[1] = {{ 175 | .server_index = -1, 176 | .header = { 177 | .opcode = opcode, 178 | .extlen = sizeof(body), 179 | .keylen = htobe16(key_len), 180 | .bodylen = htobe32(key_len + sizeof(body)), 181 | }, 182 | .extra = &body, 183 | .key = key, 184 | }}; 185 | omcache_value_t value = {0}; 186 | size_t req_count = 1, value_count = 1; 187 | int ret = omcache_command(mc, req, &req_count, &value, &value_count, timeout_msec); 188 | if (ret == OMCACHE_OK && value_count) 189 | ret = value.status; 190 | if (valuep) 191 | *valuep = (ret == OMCACHE_OK) ? value.delta_value : 0; 192 | return ret; 193 | } 194 | 195 | int omcache_increment(omcache_t *mc, 196 | const unsigned char *key, size_t key_len, 197 | uint64_t delta, uint64_t initial, 198 | time_t expiration, uint64_t *value, 199 | int32_t timeout_msec) 200 | { 201 | return omc_ctr_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_INCREMENT), 202 | key, key_len, delta, initial, expiration, 203 | value, timeout_msec); 204 | } 205 | 206 | int omcache_decrement(omcache_t *mc, 207 | const unsigned char *key, size_t key_len, 208 | uint64_t delta, uint64_t initial, 209 | time_t expiration, uint64_t *value, 210 | int32_t timeout_msec) 211 | { 212 | return omc_ctr_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_DECREMENT), 213 | key, key_len, delta, initial, expiration, 214 | value, timeout_msec); 215 | } 216 | 217 | int omcache_delete(omcache_t *mc, 218 | const unsigned char *key, size_t key_len, 219 | int32_t timeout_msec) 220 | { 221 | omcache_req_t req[1] = {{ 222 | .server_index = -1, 223 | .header = { 224 | .opcode = QCMD(PROTOCOL_BINARY_CMD_DELETE), 225 | .keylen = htobe16(key_len), 226 | .bodylen = htobe32(key_len), 227 | }, 228 | .key = key, 229 | }}; 230 | return omcache_command_status(mc, req, timeout_msec); 231 | } 232 | 233 | int omcache_touch(omcache_t *mc, 234 | const unsigned char *key, size_t key_len, 235 | time_t expiration, int32_t timeout_msec) 236 | { 237 | uint32_t body_exp = htobe32(expiration); 238 | omcache_req_t req[1] = {{ 239 | .server_index = -1, 240 | .header = { 241 | .opcode = PROTOCOL_BINARY_CMD_TOUCH, 242 | .extlen = sizeof(body_exp), 243 | .keylen = htobe16(key_len), 244 | .bodylen = htobe32(key_len + sizeof(body_exp)), 245 | }, 246 | .extra = &body_exp, 247 | .key = key, 248 | }}; 249 | return omcache_command_status(mc, req, timeout_msec); 250 | } 251 | 252 | static int omc_get_multi_cmd(omcache_t *mc, 253 | protocol_binary_command opcode, 254 | const unsigned char **keys, 255 | size_t *key_lens, 256 | uint32_t *be_expirations, 257 | size_t key_count, 258 | omcache_req_t *requests, 259 | size_t *req_count, 260 | omcache_value_t *values, 261 | size_t *value_count, 262 | int32_t timeout_msec) 263 | { 264 | if (req_count == NULL || *req_count < key_count) 265 | return OMCACHE_INVALID; 266 | 267 | memset(requests, 0, sizeof(*requests) * *req_count); 268 | if (values && value_count) 269 | memset(values, 0, sizeof(*values) * *value_count); 270 | 271 | for (size_t i = 0; i < key_count; i ++) 272 | { 273 | requests[i].server_index = -1; 274 | requests[i].header.opcode = opcode; 275 | requests[i].header.keylen = htobe16(key_lens[i]); 276 | if (opcode == PROTOCOL_BINARY_CMD_GAT || opcode == PROTOCOL_BINARY_CMD_GATQ || 277 | opcode == PROTOCOL_BINARY_CMD_GATK || opcode == PROTOCOL_BINARY_CMD_GATKQ) 278 | { 279 | requests[i].header.extlen = sizeof(uint32_t); 280 | requests[i].extra = &be_expirations[i]; 281 | } 282 | requests[i].header.bodylen = htobe32(key_lens[i] + requests[i].header.extlen); 283 | requests[i].key = keys[i]; 284 | } 285 | return omcache_command(mc, requests, req_count, values, value_count, timeout_msec); 286 | } 287 | 288 | int omcache_get_multi(omcache_t *mc, 289 | const unsigned char **keys, 290 | size_t *key_lens, 291 | size_t key_count, 292 | omcache_req_t *requests, 293 | size_t *req_count, 294 | omcache_value_t *values, 295 | size_t *value_count, 296 | int32_t timeout_msec) 297 | { 298 | return omc_get_multi_cmd(mc, PROTOCOL_BINARY_CMD_GETKQ, 299 | keys, key_lens, NULL, key_count, 300 | requests, req_count, values, value_count, 301 | timeout_msec); 302 | } 303 | 304 | int omcache_gat_multi(omcache_t *mc, 305 | const unsigned char **keys, 306 | size_t *key_lens, 307 | time_t *expirations, 308 | size_t key_count, 309 | omcache_req_t *requests, 310 | size_t *req_count, 311 | omcache_value_t *values, 312 | size_t *value_count, 313 | int32_t timeout_msec) 314 | { 315 | uint32_t be_expirations[key_count]; 316 | for (size_t i = 0; i < key_count; i ++) 317 | be_expirations[i] = htobe32(expirations[i]); 318 | return omc_get_multi_cmd(mc, PROTOCOL_BINARY_CMD_GATKQ, 319 | keys, key_lens, be_expirations, key_count, 320 | requests, req_count, values, value_count, 321 | timeout_msec); 322 | } 323 | 324 | static int omc_get_cmd(omcache_t *mc, protocol_binary_command opcode, 325 | const unsigned char *key, size_t key_len, 326 | const unsigned char **valuep, size_t *value_len, 327 | uint32_t be_expiration, uint32_t *flags, uint64_t *cas, 328 | int32_t timeout_msec) 329 | { 330 | omcache_req_t req; 331 | omcache_value_t value = {0}; 332 | size_t req_count = 1, value_count = 1; 333 | int ret = omc_get_multi_cmd(mc, opcode, &key, &key_len, &be_expiration, 1, &req, &req_count, 334 | &value, &value_count, timeout_msec); 335 | if (value_count) 336 | ret = value.status; 337 | if (value_count == 0 && ret == OMCACHE_OK) 338 | ret = OMCACHE_NOT_FOUND; 339 | else if (ret == OMCACHE_AGAIN && valuep == NULL) 340 | ret = OMCACHE_OK; 341 | if (valuep) 342 | *valuep = value.data; 343 | if (value_len) 344 | *value_len = value.data_len; 345 | if (flags) 346 | *flags = value.flags; 347 | if (cas) 348 | *cas = value.cas; 349 | return ret; 350 | } 351 | 352 | int omcache_get(omcache_t *mc, 353 | const unsigned char *key, size_t key_len, 354 | const unsigned char **value, size_t *value_len, 355 | uint32_t *flags, uint64_t *cas, 356 | int32_t timeout_msec) 357 | { 358 | return omc_get_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_GETK), key, key_len, 359 | value, value_len, 0, flags, cas, timeout_msec); 360 | } 361 | 362 | int omcache_gat(omcache_t *mc, 363 | const unsigned char *key, size_t key_len, 364 | const unsigned char **value, size_t *value_len, 365 | time_t expiration, 366 | uint32_t *flags, uint64_t *cas, 367 | int32_t timeout_msec) 368 | { 369 | return omc_get_cmd(mc, QCMD(PROTOCOL_BINARY_CMD_GATK), key, key_len, 370 | value, value_len, htobe32(expiration), 371 | flags, cas, timeout_msec); 372 | } 373 | -------------------------------------------------------------------------------- /compat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * OMcache: portability macros and functions 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #ifndef _OMCACHE_COMPAT_H 13 | #define _OMCACHE_COMPAT_H 1 14 | 15 | #include 16 | 17 | #ifdef __GNUC__ 18 | #define omc_attribute_unused __attribute__((unused)) 19 | #else 20 | #define omc_attribute_unused 21 | #endif 22 | 23 | #define max(a,b) ({__typeof__(a) a_ = (a), b_ = (b); a_ > b_ ? a_ : b_; }) 24 | #define min(a,b) ({__typeof__(a) a_ = (a), b_ = (b); a_ < b_ ? a_ : b_; }) 25 | 26 | #ifdef __linux__ 27 | #include 28 | #elif defined(__APPLE__) 29 | #include 30 | #define be16toh(x) OSSwapBigToHostInt16(x) 31 | #define be32toh(x) OSSwapBigToHostInt32(x) 32 | #define be64toh(x) OSSwapBigToHostInt64(x) 33 | #define htobe16(x) OSSwapHostToBigInt16(x) 34 | #define htobe32(x) OSSwapHostToBigInt32(x) 35 | #define htobe64(x) OSSwapHostToBigInt64(x) 36 | #elif defined(__sparc__) || defined(__sparc) 37 | #define be16toh(x) (x) 38 | #define be32toh(x) (x) 39 | #define be64toh(x) (x) 40 | #define htobe16(x) (x) 41 | #define htobe32(x) (x) 42 | #define htobe64(x) (x) 43 | #endif 44 | 45 | // MSG_NOSIGNAL suppresses SIGPIPE on send(), if we don't have it, well, 46 | // let's hope the application set SIGPIPE handler to ignore 47 | #ifndef MSG_NOSIGNAL 48 | #define MSG_NOSIGNAL 0 49 | #endif 50 | 51 | // clock_gettime is available if _POSIX_TIMERS is defined to >= 1 52 | #if !(defined(_POSIX_TIMERS) && _POSIX_TIMERS >= 1) 53 | #include 54 | #ifndef CLOCK_REALTIME 55 | #define CLOCK_REALTIME 0 56 | #endif 57 | 58 | static omc_attribute_unused 59 | int clock_gettime(int clk_id omc_attribute_unused, struct timespec *t) 60 | { 61 | struct timeval tv; 62 | int rv = gettimeofday(&tv, NULL); 63 | if (rv) 64 | return rv; 65 | t->tv_sec = tv.tv_sec; 66 | t->tv_nsec = tv.tv_usec * 1000; 67 | return 0; 68 | } 69 | #endif // !_POSIX_TIMERS 70 | 71 | // strndup is defined in POSIX.1-2008, GNU had it before, OS X since 10.7 72 | #if !(defined(_GNU_SOURCE) || _POSIX_VERSION >= 200809L || defined(__APPLE__)) 73 | #include 74 | #include 75 | 76 | static omc_attribute_unused 77 | char *strndup(const char *s, size_t n) 78 | { 79 | size_t l = min(n, strlen(s)); 80 | char *r = malloc(l + 1); 81 | memcpy(r, s, l); 82 | r[l] = 0; 83 | return r; 84 | } 85 | #endif // !(_GNU_SOURCE || _POSIX_VERSION >= 200809L || __APPLE__) 86 | 87 | #endif // !_OMCACHE_COMPAT_H 88 | -------------------------------------------------------------------------------- /compat.mk: -------------------------------------------------------------------------------- 1 | UNAME_S = $(shell uname -s) 2 | CPPFLAGS ?= -Wall -Wextra 3 | CFLAGS ?= -g -O2 4 | 5 | ifeq ($(UNAME_S),Linux) 6 | SO_EXT = so 7 | SO_FLAGS = -shared -fPIC -Wl,-soname=$(SHLIB_V) -Wl,-version-script=symbol.map 8 | WITH_LIBS += -lrt 9 | WITH_CFLAGS += -std=gnu99 -D_GNU_SOURCE 10 | else ifeq ($(UNAME_S),SunOS) 11 | SO_EXT = so 12 | SO_FLAGS = -shared -fPIC -Wl,-h,$(SHLIB_V) -Wl,-M,symbol.map 13 | WITH_LIBS += -lrt -lsocket 14 | ifneq ($(shell $(CC) -V 2>&1 | grep "Sun C"),) 15 | WITH_CFLAGS += -xc99 16 | else 17 | WITH_CFLAGS += -std=gnu99 -D_XOPEN_SOURCE=600 -D__EXTENSIONS__ 18 | endif 19 | else ifeq ($(UNAME_S),Darwin) 20 | SO_EXT = dylib 21 | SO_FLAGS = -dynamiclib 22 | WITH_CFLAGS += -std=gnu99 -D_DARWIN_C_SOURCE 23 | endif 24 | 25 | ifeq ($(WITHOUT_ASYNCNS),) 26 | WITH_CFLAGS += -DWITH_ASYNCNS 27 | WITH_LIBS += -lasyncns 28 | endif 29 | 30 | %.o: %.c 31 | $(CC) $(CPPFLAGS) $(CFLAGS) $(WITH_CFLAGS) -fPIC -c $^ 32 | -------------------------------------------------------------------------------- /debian/.gitignore: -------------------------------------------------------------------------------- 1 | /*.log 2 | /*.substvars 3 | /changelog 4 | /files 5 | /libomcache-dev/ 6 | /libomcache0/ 7 | /libomcache0.postinst.debhelper 8 | /libomcache0.postrm.debhelper 9 | /python-omcache/ 10 | /python-omcache.install 11 | /python-omcache.postinst.debhelper 12 | /python-omcache.prerm.debhelper 13 | /pyversions 14 | /stamp-makefile-build 15 | /stamp-makefile-check 16 | /stamp-makefile-install 17 | /tmp/ 18 | -------------------------------------------------------------------------------- /debian/changelog.in: -------------------------------------------------------------------------------- 1 | omcache (0.1.0) unstable; urgency=low 2 | 3 | * Initial. 4 | 5 | -- Oskari Saarenmaa Sat, 18 Oct 2014 22:26:12 +0300 6 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: omcache 2 | Section: libs 3 | Priority: extra 4 | Maintainer: Oskari Saarenmaa 5 | Build-Depends: debhelper (>= 8), python-support, check, libasyncns-dev 6 | Standards-Version: 3.9.5 7 | Homepage: https://github.com/ohmu/omcache/ 8 | 9 | Package: libomcache0 10 | Architecture: any 11 | Section: libs 12 | Depends: ${shlibs:Depends}, ${misc:Depends} 13 | Description: memcache client library 14 | OMcache is a low level C library for accessing memcached servers. The goals 15 | of the OMcache project are stable API and ABI and 'easy' integration into 16 | complex applications and systems; OMcache specifically does not mask any 17 | signals or call any blocking functions. 18 | 19 | Package: libomcache-dev 20 | Architecture: any 21 | Section: libdevel 22 | Depends: ${shlibs:Depends}, ${misc:Depends}, libomcache0 (= ${binary:Version}) 23 | Description: development files for omcache 24 | Development libraries and headers for the OMcache memcache client library. 25 | 26 | Package: python-omcache 27 | Architecture: all 28 | Section: python 29 | Depends: ${shlibs:Depends}, ${misc:Depends}, 30 | libomcache0 (= ${binary:Version}), python-cffi 31 | Description: memcached client library for python 32 | Python bindings for the OMcache memcache client library. 33 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://dep.debian.net/deps/dep5 2 | Upstream-Name: omcache 3 | Source: https://github.com/ohmu/omcache/ 4 | 5 | Files: * 6 | Copyright: 2013-2014 Oskari Saarenmaa 7 | License: Apache-2.0 8 | 9 | License: Apache-2.0 10 | On Debian GNU/Linux system you can find the complete text of the 11 | Apache 2.0 license in '/usr/share/common-licenses/Apache-2.0'. 12 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | README.rst 2 | NEWS 3 | -------------------------------------------------------------------------------- /debian/libomcache-dev.install: -------------------------------------------------------------------------------- 1 | usr/include/omcache.h 2 | usr/include/omcache_cdef.h 3 | usr/include/omcache_libmemcached.h 4 | usr/lib/libomcache.so 5 | -------------------------------------------------------------------------------- /debian/libomcache0.install: -------------------------------------------------------------------------------- 1 | usr/lib/libomcache.so.0* 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | DEB_MAKE_INSTALL_TARGET = install DESTDIR=$(DEB_DESTDIR) PREFIX=/usr 4 | DEB_MAKE_CHECK_TARGET = check 5 | 6 | include /usr/share/cdbs/1/rules/debhelper.mk 7 | include /usr/share/cdbs/1/class/makefile.mk 8 | 9 | install/python-omcache:: 10 | pyversions -s | sed -e 's-python--g' -e 's- -,-g' > $(CURDIR)/debian/pyversions 11 | $(RM) $(CURDIR)/debian/python-omcache.install 12 | for pyver in $(shell pyversions -s); do \ 13 | mkdir -p $(CURDIR)/debian/tmp/usr/lib/$$pyver/site-packages; \ 14 | install --mode=0644 omcache_cdef.h omcache.py omcache_pylibmc.py \ 15 | $(CURDIR)/debian/tmp/usr/lib/$$pyver/site-packages/; \ 16 | echo usr/lib/$$pyver/site-packages/omcache_cdef.h >> $(CURDIR)/debian/python-omcache.install; \ 17 | echo usr/lib/$$pyver/site-packages/omcache.py >> $(CURDIR)/debian/python-omcache.install; \ 18 | echo usr/lib/$$pyver/site-packages/omcache_pylibmc.py >> $(CURDIR)/debian/python-omcache.install; \ 19 | done 20 | 21 | binary-install/python-omcache:: 22 | dh_pysupport -p$(cdbs_curpkg) 23 | 24 | clean:: 25 | $(RM) debian/pyversions debian/python-omcache.install 26 | -------------------------------------------------------------------------------- /dist.c: -------------------------------------------------------------------------------- 1 | /* 2 | * OMcache: distribution functions 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include "omcache_priv.h" 16 | 17 | 18 | omc_hidden uint32_t omc_hash_jenkins_oat(const unsigned char *key, size_t key_len) 19 | { 20 | // http://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time 21 | uint32_t hash, i; 22 | for (hash = i = 0; i < key_len; i ++) 23 | { 24 | hash += key[i]; 25 | hash += (hash << 10); 26 | hash ^= (hash >> 6); 27 | } 28 | hash += (hash << 3); 29 | hash ^= (hash >> 11); 30 | hash += (hash << 15); 31 | return hash; 32 | } 33 | 34 | static uint32_t omc_hash_md5_32(const unsigned char *key, size_t key_len) 35 | { 36 | // truncate md5 hash to 32 bits like libmemcached's hashkit 37 | unsigned char md5buf[16]; 38 | omc_hash_md5((unsigned char *) key, key_len, md5buf); 39 | return ((uint32_t) md5buf[3] << 24) | ((uint32_t) md5buf[2] << 16) | 40 | ((uint32_t) md5buf[1] << 8) | ((uint32_t) md5buf[0] << 0); 41 | } 42 | 43 | static size_t omc_ketama_point_name(const char *hostname, const char *portname, uint32_t point, char *namebuf) 44 | { 45 | // libmemcached ketama appends port number to hostname if it's not the default (11211) 46 | bool with_port = strcmp(portname, MC_PORT) != 0; 47 | return sprintf(namebuf, "%s%s%s-%u", 48 | hostname, with_port ? ":" : "", with_port ? portname : "", point); 49 | } 50 | 51 | static uint32_t omc_ketama_jenkins_oat(const char *hostname, const char *portname, 52 | uint32_t point, uint32_t *hashes) 53 | { 54 | char name[strlen(hostname) + strlen(portname) + 16]; 55 | size_t name_len = omc_ketama_point_name(hostname, portname, point, name); 56 | hashes[0] = omc_hash_jenkins_oat((unsigned char *) name, name_len); 57 | return 1; 58 | } 59 | 60 | static uint32_t omc_ketama_md5_libmcd_weighted(const char *hostname, const char *portname, 61 | uint32_t point, uint32_t *hashes) 62 | { 63 | // libmemcached ketama grabs four different hash values from a single md5 64 | // buffer in 'weighted ketama' mode 65 | char name[strlen(hostname) + strlen(portname) + 16]; 66 | size_t name_len = omc_ketama_point_name(hostname, portname, point, name); 67 | unsigned char md5buf[16]; 68 | omc_hash_md5((unsigned char *) name, name_len, md5buf); 69 | for (int offset = 0; offset < 4; offset ++) 70 | hashes[offset] = 71 | ((uint32_t) md5buf[3 + offset * 4] << 24) | 72 | ((uint32_t) md5buf[2 + offset * 4] << 16) | 73 | ((uint32_t) md5buf[1 + offset * 4] << 8) | 74 | ((uint32_t) md5buf[0 + offset * 4] << 0); 75 | return 4; 76 | } 77 | 78 | omcache_dist_t omcache_dist_libmemcached_ketama = { 79 | .omcache_version = OMCACHE_VERSION, 80 | .points_per_server = 100, 81 | .entries_per_point = 1, 82 | .point_hash_func = omc_ketama_jenkins_oat, 83 | .key_hash_func = omc_hash_jenkins_oat, 84 | }; 85 | 86 | omcache_dist_t omcache_dist_libmemcached_ketama_weighted = { 87 | .omcache_version = OMCACHE_VERSION, 88 | .points_per_server = 40, 89 | .entries_per_point = 4, 90 | .point_hash_func = omc_ketama_md5_libmcd_weighted, 91 | .key_hash_func = omc_hash_md5_32, 92 | }; 93 | 94 | omcache_dist_t omcache_dist_libmemcached_ketama_pre1010 = { 95 | .omcache_version = OMCACHE_VERSION, 96 | .points_per_server = 40, 97 | .entries_per_point = 4, 98 | .point_hash_func = omc_ketama_md5_libmcd_weighted, 99 | .key_hash_func = omc_hash_jenkins_oat, 100 | }; 101 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | html/ 2 | latex/ 3 | omcache_c_api_xml/ 4 | index.rst 5 | -------------------------------------------------------------------------------- /doc/Doxyfile: -------------------------------------------------------------------------------- 1 | INPUT = ../omcache_cdef.h 2 | EXTRACT_ALL = YES 3 | GENERATE_XML = YES 4 | XML_OUTPUT = omcache_c_api_xml 5 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | include ../version.mk 2 | 3 | export OMCACHE_VERSION = $(short_ver) 4 | export OMCACHE_VERSION_FULL = $(long_ver) 5 | 6 | all: 7 | doxygen 8 | sphinx-build . html 9 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | if os.environ.get('READTHEDOCS', None) == 'True': 4 | import subprocess, sys 5 | subprocess.call(['doxygen']) 6 | subprocess.call(["git", "clone", "--depth", "1", "--branch", "v3.2.0", "https://github.com/michaeljones/breathe.git", "../breathe-git"]) 7 | sys.path.append("../breathe-git") 8 | version = subprocess.check_output(["git", "describe", "--abbrev=0"]) 9 | release = subprocess.check_output(["git", "describe", "--long"]) 10 | else: 11 | version = os.getenv("OMCACHE_VERSION") 12 | release = os.getenv("OMCACHE_VERSION_FULL") 13 | 14 | # generate index.rst from readme, skip the badges, add toc 15 | readme_lines = open("../README.rst").read().splitlines() 16 | readme_content = "\n".join(readme_lines[9:]) 17 | beef = """ 18 | ======= 19 | OMcache 20 | ======= 21 | 22 | .. toctree:: 23 | :maxdepth: 2 24 | 25 | omcache_cdef 26 | 27 | Introduction 28 | ============ 29 | """ + readme_content 30 | open("index.rst", "w").write(beef) 31 | 32 | 33 | project = 'OMcache' 34 | copyright = '2013-2014, Oskari Saarenmaa' 35 | 36 | extensions = ['breathe'] 37 | breathe_projects = {'omcache_c_api': 'omcache_c_api_xml'} 38 | breathe_default_project = 'omcache_c_api' 39 | source_suffix = '.rst' 40 | master_doc = 'index' 41 | pygments_style = 'sphinx' 42 | highlight_language = 'c' 43 | primary_domain = 'c' 44 | html_use_index = False 45 | -------------------------------------------------------------------------------- /doc/omcache_cdef.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | C API Reference 3 | =============== 4 | 5 | .. doxygenindex:: 6 | :project: omcache_c_api 7 | -------------------------------------------------------------------------------- /md5.c: -------------------------------------------------------------------------------- 1 | /* 2 | * MD5 hashing for OMcache. 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * The OMcache bits in this file are under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | * Most of the code below was copied from 11 | * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 12 | * and modified to not export the proper MD5 interfaces but just expose a 13 | * Init+Update+Final function for use in OMcache's consistent hashing withe 14 | * libmemcached compatibility mode. 15 | * 16 | */ 17 | 18 | #include "omcache_priv.h" 19 | 20 | /* 21 | * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. 22 | * MD5 Message-Digest Algorithm (RFC 1321). 23 | * 24 | * Homepage: 25 | * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 26 | * 27 | * Author: 28 | * Alexander Peslyak, better known as Solar Designer 29 | * 30 | * This software was written by Alexander Peslyak in 2001. No copyright is 31 | * claimed, and the software is hereby placed in the public domain. 32 | * In case this attempt to disclaim copyright and place the software in the 33 | * public domain is deemed null and void, then the software is 34 | * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the 35 | * general public under the following terms: 36 | * 37 | * Redistribution and use in source and binary forms, with or without 38 | * modification, are permitted. 39 | * 40 | * There's ABSOLUTELY NO WARRANTY, express or implied. 41 | * 42 | * (This is a heavily cut-down "BSD license".) 43 | * 44 | * This differs from Colin Plumb's older public domain implementation in that 45 | * no exactly 32-bit integer data type is required (any 32-bit or wider 46 | * unsigned integer data type will do), there's no compile-time endianness 47 | * configuration, and the function prototypes match OpenSSL's. No code from 48 | * Colin Plumb's implementation has been reused; this comment merely compares 49 | * the properties of the two independent implementations. 50 | * 51 | * The primary goals of this implementation are portability and ease of use. 52 | * It is meant to be fast, but not as fast as possible. Some known 53 | * optimizations are not included to reduce source code size and avoid 54 | * compile-time configuration. 55 | */ 56 | 57 | #include 58 | 59 | /* Any 32-bit or wider unsigned integer data type will do */ 60 | typedef unsigned int MD5_u32plus; 61 | 62 | typedef struct { 63 | MD5_u32plus lo, hi; 64 | MD5_u32plus a, b, c, d; 65 | unsigned char buffer[64]; 66 | MD5_u32plus block[16]; 67 | } MD5_CTX; 68 | 69 | /* 70 | * The basic MD5 functions. 71 | * 72 | * F and G are optimized compared to their RFC 1321 definitions for 73 | * architectures that lack an AND-NOT instruction, just like in Colin Plumb's 74 | * implementation. 75 | */ 76 | #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) 77 | #define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y)))) 78 | #define H(x, y, z) (((x) ^ (y)) ^ (z)) 79 | #define H2(x, y, z) ((x) ^ ((y) ^ (z))) 80 | #define I(x, y, z) ((y) ^ ((x) | ~(z))) 81 | 82 | /* 83 | * The MD5 transformation for all four rounds. 84 | */ 85 | #define STEP(f, a, b, c, d, x, t, s) \ 86 | (a) += f((b), (c), (d)) + (x) + (t); \ 87 | (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \ 88 | (a) += (b); 89 | 90 | /* 91 | * SET reads 4 input bytes in little-endian byte order and stores them 92 | * in a properly aligned word in host byte order. 93 | * 94 | * The check for little-endian architectures that tolerate unaligned 95 | * memory accesses is just an optimization. Nothing will break if it 96 | * doesn't work. 97 | */ 98 | #if defined(__i386__) || defined(__x86_64__) || defined(__vax__) 99 | #define SET(n) \ 100 | (*(MD5_u32plus *)&ptr[(n) * 4]) 101 | #define GET(n) \ 102 | SET(n) 103 | #else 104 | #define SET(n) \ 105 | (ctx->block[(n)] = \ 106 | (MD5_u32plus)ptr[(n) * 4] | \ 107 | ((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \ 108 | ((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \ 109 | ((MD5_u32plus)ptr[(n) * 4 + 3] << 24)) 110 | #define GET(n) \ 111 | (ctx->block[(n)]) 112 | #endif 113 | 114 | /* 115 | * This processes one or more 64-byte data blocks, but does NOT update 116 | * the bit counters. There are no alignment requirements. 117 | */ 118 | static const void *body(MD5_CTX *ctx, const void *data, unsigned long size) 119 | { 120 | const unsigned char *ptr; 121 | MD5_u32plus a, b, c, d; 122 | MD5_u32plus saved_a, saved_b, saved_c, saved_d; 123 | 124 | ptr = (const unsigned char *)data; 125 | 126 | a = ctx->a; 127 | b = ctx->b; 128 | c = ctx->c; 129 | d = ctx->d; 130 | 131 | do { 132 | saved_a = a; 133 | saved_b = b; 134 | saved_c = c; 135 | saved_d = d; 136 | 137 | /* Round 1 */ 138 | STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7) 139 | STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12) 140 | STEP(F, c, d, a, b, SET(2), 0x242070db, 17) 141 | STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22) 142 | STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7) 143 | STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12) 144 | STEP(F, c, d, a, b, SET(6), 0xa8304613, 17) 145 | STEP(F, b, c, d, a, SET(7), 0xfd469501, 22) 146 | STEP(F, a, b, c, d, SET(8), 0x698098d8, 7) 147 | STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12) 148 | STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17) 149 | STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22) 150 | STEP(F, a, b, c, d, SET(12), 0x6b901122, 7) 151 | STEP(F, d, a, b, c, SET(13), 0xfd987193, 12) 152 | STEP(F, c, d, a, b, SET(14), 0xa679438e, 17) 153 | STEP(F, b, c, d, a, SET(15), 0x49b40821, 22) 154 | 155 | /* Round 2 */ 156 | STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5) 157 | STEP(G, d, a, b, c, GET(6), 0xc040b340, 9) 158 | STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14) 159 | STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20) 160 | STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5) 161 | STEP(G, d, a, b, c, GET(10), 0x02441453, 9) 162 | STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14) 163 | STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20) 164 | STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5) 165 | STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9) 166 | STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14) 167 | STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20) 168 | STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5) 169 | STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9) 170 | STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14) 171 | STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20) 172 | 173 | /* Round 3 */ 174 | STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4) 175 | STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11) 176 | STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16) 177 | STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23) 178 | STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4) 179 | STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11) 180 | STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16) 181 | STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23) 182 | STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4) 183 | STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11) 184 | STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16) 185 | STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23) 186 | STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4) 187 | STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11) 188 | STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16) 189 | STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23) 190 | 191 | /* Round 4 */ 192 | STEP(I, a, b, c, d, GET(0), 0xf4292244, 6) 193 | STEP(I, d, a, b, c, GET(7), 0x432aff97, 10) 194 | STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15) 195 | STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21) 196 | STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6) 197 | STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10) 198 | STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15) 199 | STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21) 200 | STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6) 201 | STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10) 202 | STEP(I, c, d, a, b, GET(6), 0xa3014314, 15) 203 | STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21) 204 | STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6) 205 | STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10) 206 | STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15) 207 | STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21) 208 | 209 | a += saved_a; 210 | b += saved_b; 211 | c += saved_c; 212 | d += saved_d; 213 | 214 | ptr += 64; 215 | } while (size -= 64); 216 | 217 | ctx->a = a; 218 | ctx->b = b; 219 | ctx->c = c; 220 | ctx->d = d; 221 | 222 | return ptr; 223 | } 224 | 225 | static void MD5_Init(MD5_CTX *ctx) 226 | { 227 | ctx->a = 0x67452301; 228 | ctx->b = 0xefcdab89; 229 | ctx->c = 0x98badcfe; 230 | ctx->d = 0x10325476; 231 | 232 | ctx->lo = 0; 233 | ctx->hi = 0; 234 | } 235 | 236 | static void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size) 237 | { 238 | MD5_u32plus saved_lo; 239 | unsigned long used, available; 240 | 241 | saved_lo = ctx->lo; 242 | if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo) 243 | ctx->hi++; 244 | ctx->hi += size >> 29; 245 | 246 | used = saved_lo & 0x3f; 247 | 248 | if (used) { 249 | available = 64 - used; 250 | 251 | if (size < available) { 252 | memcpy(&ctx->buffer[used], data, size); 253 | return; 254 | } 255 | 256 | memcpy(&ctx->buffer[used], data, available); 257 | data = (const unsigned char *)data + available; 258 | size -= available; 259 | body(ctx, ctx->buffer, 64); 260 | } 261 | 262 | if (size >= 64) { 263 | data = body(ctx, data, size & ~(unsigned long)0x3f); 264 | size &= 0x3f; 265 | } 266 | 267 | memcpy(ctx->buffer, data, size); 268 | } 269 | 270 | static void MD5_Final(unsigned char *result, MD5_CTX *ctx) 271 | { 272 | unsigned long used, available; 273 | 274 | used = ctx->lo & 0x3f; 275 | 276 | ctx->buffer[used++] = 0x80; 277 | 278 | available = 64 - used; 279 | 280 | if (available < 8) { 281 | memset(&ctx->buffer[used], 0, available); 282 | body(ctx, ctx->buffer, 64); 283 | used = 0; 284 | available = 64; 285 | } 286 | 287 | memset(&ctx->buffer[used], 0, available - 8); 288 | 289 | ctx->lo <<= 3; 290 | ctx->buffer[56] = ctx->lo; 291 | ctx->buffer[57] = ctx->lo >> 8; 292 | ctx->buffer[58] = ctx->lo >> 16; 293 | ctx->buffer[59] = ctx->lo >> 24; 294 | ctx->buffer[60] = ctx->hi; 295 | ctx->buffer[61] = ctx->hi >> 8; 296 | ctx->buffer[62] = ctx->hi >> 16; 297 | ctx->buffer[63] = ctx->hi >> 24; 298 | 299 | body(ctx, ctx->buffer, 64); 300 | 301 | result[0] = ctx->a; 302 | result[1] = ctx->a >> 8; 303 | result[2] = ctx->a >> 16; 304 | result[3] = ctx->a >> 24; 305 | result[4] = ctx->b; 306 | result[5] = ctx->b >> 8; 307 | result[6] = ctx->b >> 16; 308 | result[7] = ctx->b >> 24; 309 | result[8] = ctx->c; 310 | result[9] = ctx->c >> 8; 311 | result[10] = ctx->c >> 16; 312 | result[11] = ctx->c >> 24; 313 | result[12] = ctx->d; 314 | result[13] = ctx->d >> 8; 315 | result[14] = ctx->d >> 16; 316 | result[15] = ctx->d >> 24; 317 | 318 | memset(ctx, 0, sizeof(*ctx)); 319 | } 320 | 321 | void omc_hash_md5(const unsigned char *key, size_t key_len, unsigned char *buf) 322 | { 323 | MD5_CTX ctx; 324 | MD5_Init(&ctx); 325 | MD5_Update(&ctx, key, key_len); 326 | MD5_Final(buf, &ctx); 327 | } 328 | -------------------------------------------------------------------------------- /memcached_protocol_binary.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) <2008>, Sun Microsystems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * * Redistributions of source code must retain the above copyright 8 | * notice, this list of conditions and the following disclaimer. 9 | * * Redistributions in binary form must reproduce the above copyright 10 | * notice, this list of conditions and the following disclaimer in the 11 | * documentation and/or other materials provided with the distribution. 12 | * * Neither the name of the nor the 13 | * names of its contributors may be used to endorse or promote products 14 | * derived from this software without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY SUN MICROSYSTEMS, INC. ``AS IS'' AND ANY 17 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL SUN MICROSYSTEMS, INC. BE LIABLE FOR ANY 20 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | /* 28 | * Summary: Constants used by to implement the binary protocol. 29 | * 30 | * Copy: See Copyright for the status of this software. 31 | * 32 | * Author: Trond Norbye 33 | */ 34 | 35 | #ifndef PROTOCOL_BINARY_H 36 | #define PROTOCOL_BINARY_H 37 | 38 | /** 39 | * This file contains definitions of the constants and packet formats 40 | * defined in the binary specification. Please note that you _MUST_ remember 41 | * to convert each multibyte field to / from network byte order to / from 42 | * host order. 43 | */ 44 | #ifdef __cplusplus 45 | extern "C" 46 | { 47 | #endif 48 | 49 | /** 50 | * Definition of the legal "magic" values used in a packet. 51 | * See section 3.1 Magic byte 52 | */ 53 | typedef enum { 54 | PROTOCOL_BINARY_REQ = 0x80, 55 | PROTOCOL_BINARY_RES = 0x81 56 | } protocol_binary_magic; 57 | 58 | /** 59 | * Definition of the valid response status numbers. 60 | * See section 3.2 Response Status 61 | */ 62 | typedef enum { 63 | PROTOCOL_BINARY_RESPONSE_SUCCESS = 0x00, 64 | PROTOCOL_BINARY_RESPONSE_KEY_ENOENT = 0x01, 65 | PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS = 0x02, 66 | PROTOCOL_BINARY_RESPONSE_E2BIG = 0x03, 67 | PROTOCOL_BINARY_RESPONSE_EINVAL = 0x04, 68 | PROTOCOL_BINARY_RESPONSE_NOT_STORED = 0x05, 69 | PROTOCOL_BINARY_RESPONSE_DELTA_BADVAL = 0x06, 70 | PROTOCOL_BINARY_RESPONSE_AUTH_ERROR = 0x20, 71 | PROTOCOL_BINARY_RESPONSE_AUTH_CONTINUE = 0x21, 72 | PROTOCOL_BINARY_RESPONSE_UNKNOWN_COMMAND = 0x81, 73 | PROTOCOL_BINARY_RESPONSE_ENOMEM = 0x82 74 | } protocol_binary_response_status; 75 | 76 | /** 77 | * Defintion of the different command opcodes. 78 | * See section 3.3 Command Opcodes 79 | */ 80 | typedef enum { 81 | PROTOCOL_BINARY_CMD_GET = 0x00, 82 | PROTOCOL_BINARY_CMD_SET = 0x01, 83 | PROTOCOL_BINARY_CMD_ADD = 0x02, 84 | PROTOCOL_BINARY_CMD_REPLACE = 0x03, 85 | PROTOCOL_BINARY_CMD_DELETE = 0x04, 86 | PROTOCOL_BINARY_CMD_INCREMENT = 0x05, 87 | PROTOCOL_BINARY_CMD_DECREMENT = 0x06, 88 | PROTOCOL_BINARY_CMD_QUIT = 0x07, 89 | PROTOCOL_BINARY_CMD_FLUSH = 0x08, 90 | PROTOCOL_BINARY_CMD_GETQ = 0x09, 91 | PROTOCOL_BINARY_CMD_NOOP = 0x0a, 92 | PROTOCOL_BINARY_CMD_VERSION = 0x0b, 93 | PROTOCOL_BINARY_CMD_GETK = 0x0c, 94 | PROTOCOL_BINARY_CMD_GETKQ = 0x0d, 95 | PROTOCOL_BINARY_CMD_APPEND = 0x0e, 96 | PROTOCOL_BINARY_CMD_PREPEND = 0x0f, 97 | PROTOCOL_BINARY_CMD_STAT = 0x10, 98 | PROTOCOL_BINARY_CMD_SETQ = 0x11, 99 | PROTOCOL_BINARY_CMD_ADDQ = 0x12, 100 | PROTOCOL_BINARY_CMD_REPLACEQ = 0x13, 101 | PROTOCOL_BINARY_CMD_DELETEQ = 0x14, 102 | PROTOCOL_BINARY_CMD_INCREMENTQ = 0x15, 103 | PROTOCOL_BINARY_CMD_DECREMENTQ = 0x16, 104 | PROTOCOL_BINARY_CMD_QUITQ = 0x17, 105 | PROTOCOL_BINARY_CMD_FLUSHQ = 0x18, 106 | PROTOCOL_BINARY_CMD_APPENDQ = 0x19, 107 | PROTOCOL_BINARY_CMD_PREPENDQ = 0x1a, 108 | PROTOCOL_BINARY_CMD_TOUCH = 0x1c, 109 | PROTOCOL_BINARY_CMD_GAT = 0x1d, 110 | PROTOCOL_BINARY_CMD_GATQ = 0x1e, 111 | PROTOCOL_BINARY_CMD_GATK = 0x23, 112 | PROTOCOL_BINARY_CMD_GATKQ = 0x24, 113 | 114 | PROTOCOL_BINARY_CMD_SASL_LIST_MECHS = 0x20, 115 | PROTOCOL_BINARY_CMD_SASL_AUTH = 0x21, 116 | PROTOCOL_BINARY_CMD_SASL_STEP = 0x22, 117 | 118 | /* These commands are used for range operations and exist within 119 | * this header for use in other projects. Range operations are 120 | * not expected to be implemented in the memcached server itself. 121 | */ 122 | PROTOCOL_BINARY_CMD_RGET = 0x30, 123 | PROTOCOL_BINARY_CMD_RSET = 0x31, 124 | PROTOCOL_BINARY_CMD_RSETQ = 0x32, 125 | PROTOCOL_BINARY_CMD_RAPPEND = 0x33, 126 | PROTOCOL_BINARY_CMD_RAPPENDQ = 0x34, 127 | PROTOCOL_BINARY_CMD_RPREPEND = 0x35, 128 | PROTOCOL_BINARY_CMD_RPREPENDQ = 0x36, 129 | PROTOCOL_BINARY_CMD_RDELETE = 0x37, 130 | PROTOCOL_BINARY_CMD_RDELETEQ = 0x38, 131 | PROTOCOL_BINARY_CMD_RINCR = 0x39, 132 | PROTOCOL_BINARY_CMD_RINCRQ = 0x3a, 133 | PROTOCOL_BINARY_CMD_RDECR = 0x3b, 134 | PROTOCOL_BINARY_CMD_RDECRQ = 0x3c 135 | /* End Range operations */ 136 | 137 | } protocol_binary_command; 138 | 139 | /** 140 | * Definition of the data types in the packet 141 | * See section 3.4 Data Types 142 | */ 143 | typedef enum { 144 | PROTOCOL_BINARY_RAW_BYTES = 0x00 145 | } protocol_binary_datatypes; 146 | 147 | /** 148 | * Definition of the header structure for a request packet. 149 | * See section 2 150 | */ 151 | typedef union { 152 | struct { 153 | uint8_t magic; 154 | uint8_t opcode; 155 | uint16_t keylen; 156 | uint8_t extlen; 157 | uint8_t datatype; 158 | uint16_t reserved; 159 | uint32_t bodylen; 160 | uint32_t opaque; 161 | uint64_t cas; 162 | } request; 163 | uint8_t bytes[24]; 164 | } protocol_binary_request_header; 165 | 166 | /** 167 | * Definition of the header structure for a response packet. 168 | * See section 2 169 | */ 170 | typedef union { 171 | struct { 172 | uint8_t magic; 173 | uint8_t opcode; 174 | uint16_t keylen; 175 | uint8_t extlen; 176 | uint8_t datatype; 177 | uint16_t status; 178 | uint32_t bodylen; 179 | uint32_t opaque; 180 | uint64_t cas; 181 | } response; 182 | uint8_t bytes[24]; 183 | } protocol_binary_response_header; 184 | 185 | /** 186 | * Definition of a request-packet containing no extras 187 | */ 188 | typedef union { 189 | struct { 190 | protocol_binary_request_header header; 191 | } message; 192 | uint8_t bytes[sizeof(protocol_binary_request_header)]; 193 | } protocol_binary_request_no_extras; 194 | 195 | /** 196 | * Definition of a response-packet containing no extras 197 | */ 198 | typedef union { 199 | struct { 200 | protocol_binary_response_header header; 201 | } message; 202 | uint8_t bytes[sizeof(protocol_binary_response_header)]; 203 | } protocol_binary_response_no_extras; 204 | 205 | /** 206 | * Definition of the packet used by the get, getq, getk and getkq command. 207 | * See section 4 208 | */ 209 | typedef protocol_binary_request_no_extras protocol_binary_request_get; 210 | typedef protocol_binary_request_no_extras protocol_binary_request_getq; 211 | typedef protocol_binary_request_no_extras protocol_binary_request_getk; 212 | typedef protocol_binary_request_no_extras protocol_binary_request_getkq; 213 | 214 | /** 215 | * Definition of the packet returned from a successful get, getq, getk and 216 | * getkq. 217 | * See section 4 218 | */ 219 | typedef union { 220 | struct { 221 | protocol_binary_response_header header; 222 | struct { 223 | uint32_t flags; 224 | } body; 225 | } message; 226 | uint8_t bytes[sizeof(protocol_binary_response_header) + 4]; 227 | } protocol_binary_response_get; 228 | 229 | typedef protocol_binary_response_get protocol_binary_response_getq; 230 | typedef protocol_binary_response_get protocol_binary_response_getk; 231 | typedef protocol_binary_response_get protocol_binary_response_getkq; 232 | 233 | /** 234 | * Definition of the packet used by the delete command 235 | * See section 4 236 | */ 237 | typedef protocol_binary_request_no_extras protocol_binary_request_delete; 238 | 239 | /** 240 | * Definition of the packet returned by the delete command 241 | * See section 4 242 | */ 243 | typedef protocol_binary_response_no_extras protocol_binary_response_delete; 244 | 245 | /** 246 | * Definition of the packet used by the flush command 247 | * See section 4 248 | * Please note that the expiration field is optional, so remember to see 249 | * check the header.bodysize to see if it is present. 250 | */ 251 | typedef union { 252 | struct { 253 | protocol_binary_request_header header; 254 | struct { 255 | uint32_t expiration; 256 | } body; 257 | } message; 258 | uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; 259 | } protocol_binary_request_flush; 260 | 261 | /** 262 | * Definition of the packet returned by the flush command 263 | * See section 4 264 | */ 265 | typedef protocol_binary_response_no_extras protocol_binary_response_flush; 266 | 267 | /** 268 | * Definition of the packet used by set, add and replace 269 | * See section 4 270 | */ 271 | typedef union { 272 | struct { 273 | protocol_binary_request_header header; 274 | struct { 275 | uint32_t flags; 276 | uint32_t expiration; 277 | } body; 278 | } message; 279 | uint8_t bytes[sizeof(protocol_binary_request_header) + 8]; 280 | } protocol_binary_request_set; 281 | typedef protocol_binary_request_set protocol_binary_request_add; 282 | typedef protocol_binary_request_set protocol_binary_request_replace; 283 | 284 | /** 285 | * Definition of the packet returned by set, add and replace 286 | * See section 4 287 | */ 288 | typedef protocol_binary_response_no_extras protocol_binary_response_set; 289 | typedef protocol_binary_response_no_extras protocol_binary_response_add; 290 | typedef protocol_binary_response_no_extras protocol_binary_response_replace; 291 | 292 | /** 293 | * Definition of the noop packet 294 | * See section 4 295 | */ 296 | typedef protocol_binary_request_no_extras protocol_binary_request_noop; 297 | 298 | /** 299 | * Definition of the packet returned by the noop command 300 | * See section 4 301 | */ 302 | typedef protocol_binary_response_no_extras protocol_binary_response_noop; 303 | 304 | /** 305 | * Definition of the structure used by the increment and decrement 306 | * command. 307 | * See section 4 308 | */ 309 | typedef union { 310 | struct { 311 | protocol_binary_request_header header; 312 | struct { 313 | uint64_t delta; 314 | uint64_t initial; 315 | uint32_t expiration; 316 | } body; 317 | } message; 318 | uint8_t bytes[sizeof(protocol_binary_request_header) + 20]; 319 | } protocol_binary_request_incr; 320 | typedef protocol_binary_request_incr protocol_binary_request_decr; 321 | 322 | /** 323 | * Definition of the response from an incr or decr command 324 | * command. 325 | * See section 4 326 | */ 327 | typedef union { 328 | struct { 329 | protocol_binary_response_header header; 330 | struct { 331 | uint64_t value; 332 | } body; 333 | } message; 334 | uint8_t bytes[sizeof(protocol_binary_response_header) + 8]; 335 | } protocol_binary_response_incr; 336 | typedef protocol_binary_response_incr protocol_binary_response_decr; 337 | 338 | /** 339 | * Definition of the quit 340 | * See section 4 341 | */ 342 | typedef protocol_binary_request_no_extras protocol_binary_request_quit; 343 | 344 | /** 345 | * Definition of the packet returned by the quit command 346 | * See section 4 347 | */ 348 | typedef protocol_binary_response_no_extras protocol_binary_response_quit; 349 | 350 | /** 351 | * Definition of the packet used by append and prepend command 352 | * See section 4 353 | */ 354 | typedef protocol_binary_request_no_extras protocol_binary_request_append; 355 | typedef protocol_binary_request_no_extras protocol_binary_request_prepend; 356 | 357 | /** 358 | * Definition of the packet returned from a successful append or prepend 359 | * See section 4 360 | */ 361 | typedef protocol_binary_response_no_extras protocol_binary_response_append; 362 | typedef protocol_binary_response_no_extras protocol_binary_response_prepend; 363 | 364 | /** 365 | * Definition of the packet used by the version command 366 | * See section 4 367 | */ 368 | typedef protocol_binary_request_no_extras protocol_binary_request_version; 369 | 370 | /** 371 | * Definition of the packet returned from a successful version command 372 | * See section 4 373 | */ 374 | typedef protocol_binary_response_no_extras protocol_binary_response_version; 375 | 376 | 377 | /** 378 | * Definition of the packet used by the stats command. 379 | * See section 4 380 | */ 381 | typedef protocol_binary_request_no_extras protocol_binary_request_stats; 382 | 383 | /** 384 | * Definition of the packet returned from a successful stats command 385 | * See section 4 386 | */ 387 | typedef protocol_binary_response_no_extras protocol_binary_response_stats; 388 | 389 | /** 390 | * Definition of the packet used by the touch command. 391 | */ 392 | typedef union { 393 | struct { 394 | protocol_binary_request_header header; 395 | struct { 396 | uint32_t expiration; 397 | } body; 398 | } message; 399 | uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; 400 | } protocol_binary_request_touch; 401 | 402 | /** 403 | * Definition of the packet returned from the touch command 404 | */ 405 | typedef protocol_binary_response_no_extras protocol_binary_response_touch; 406 | 407 | /** 408 | * Definition of the packet used by the GAT(Q) command. 409 | */ 410 | typedef union { 411 | struct { 412 | protocol_binary_request_header header; 413 | struct { 414 | uint32_t expiration; 415 | } body; 416 | } message; 417 | uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; 418 | } protocol_binary_request_gat; 419 | 420 | typedef protocol_binary_request_gat protocol_binary_request_gatq; 421 | typedef protocol_binary_request_gat protocol_binary_request_gatk; 422 | typedef protocol_binary_request_gat protocol_binary_request_gatkq; 423 | 424 | /** 425 | * Definition of the packet returned from the GAT(Q) 426 | */ 427 | typedef protocol_binary_response_get protocol_binary_response_gat; 428 | typedef protocol_binary_response_get protocol_binary_response_gatq; 429 | typedef protocol_binary_response_get protocol_binary_response_gatk; 430 | typedef protocol_binary_response_get protocol_binary_response_gatkq; 431 | 432 | /** 433 | * Definition of a request for a range operation. 434 | * See http://code.google.com/p/memcached/wiki/RangeOps 435 | * 436 | * These types are used for range operations and exist within 437 | * this header for use in other projects. Range operations are 438 | * not expected to be implemented in the memcached server itself. 439 | */ 440 | typedef union { 441 | struct { 442 | protocol_binary_response_header header; 443 | struct { 444 | uint16_t size; 445 | uint8_t reserved; 446 | uint8_t flags; 447 | uint32_t max_results; 448 | } body; 449 | } message; 450 | uint8_t bytes[sizeof(protocol_binary_request_header) + 4]; 451 | } protocol_binary_request_rangeop; 452 | 453 | typedef protocol_binary_request_rangeop protocol_binary_request_rget; 454 | typedef protocol_binary_request_rangeop protocol_binary_request_rset; 455 | typedef protocol_binary_request_rangeop protocol_binary_request_rsetq; 456 | typedef protocol_binary_request_rangeop protocol_binary_request_rappend; 457 | typedef protocol_binary_request_rangeop protocol_binary_request_rappendq; 458 | typedef protocol_binary_request_rangeop protocol_binary_request_rprepend; 459 | typedef protocol_binary_request_rangeop protocol_binary_request_rprependq; 460 | typedef protocol_binary_request_rangeop protocol_binary_request_rdelete; 461 | typedef protocol_binary_request_rangeop protocol_binary_request_rdeleteq; 462 | typedef protocol_binary_request_rangeop protocol_binary_request_rincr; 463 | typedef protocol_binary_request_rangeop protocol_binary_request_rincrq; 464 | typedef protocol_binary_request_rangeop protocol_binary_request_rdecr; 465 | typedef protocol_binary_request_rangeop protocol_binary_request_rdecrq; 466 | 467 | #ifdef __cplusplus 468 | } 469 | #endif 470 | #endif /* PROTOCOL_BINARY_H */ 471 | -------------------------------------------------------------------------------- /omcache.h: -------------------------------------------------------------------------------- 1 | /* 2 | * OMcache - a memcached client library 3 | * 4 | * Copyright (c) 2013-2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #ifndef _OMCACHE_H 13 | #define _OMCACHE_H 1 14 | 15 | #include 16 | #include 17 | 18 | #ifdef __cplusplus 19 | extern "C" { 20 | #endif // __cplusplus 21 | 22 | #include "omcache_cdef.h" 23 | 24 | #ifdef __cplusplus 25 | } 26 | #endif // __cplusplus 27 | 28 | // CFFI can't handle defines yet 29 | #define OMCACHE_VERSION 0x00000300 // Version 0.3.0 30 | #define OMCACHE_DELTA_NO_ADD 0xffffffffu 31 | 32 | #endif // !_OMCACHE_H 33 | -------------------------------------------------------------------------------- /omcache.py: -------------------------------------------------------------------------------- 1 | # OMcache - a memcached client library 2 | # 3 | # Copyright (c) 2013-2014, Oskari Saarenmaa 4 | # All rights reserved. 5 | # 6 | # This file is under the Apache License, Version 2.0. 7 | # See the file `LICENSE` for details. 8 | 9 | from collections import namedtuple 10 | from functools import wraps 11 | from select import select as select_select, error as select_error 12 | from sys import platform, version_info 13 | import cffi 14 | import errno 15 | import logging 16 | import os 17 | import socket 18 | import time 19 | 20 | _ffi = cffi.FFI() 21 | _ffi.cdef(""" 22 | typedef long time_t; 23 | struct pollfd { 24 | int fd; /* file descriptor */ 25 | short events; /* requested events */ 26 | short revents; /* returned events */ 27 | }; 28 | """) 29 | _ffi.cdef(open(os.path.join(os.path.dirname(__file__), "omcache_cdef.h")).read()) 30 | 31 | if platform == "darwin": 32 | _oc = _ffi.dlopen("libomcache.dylib.0") 33 | else: 34 | _oc = _ffi.dlopen("libomcache.so.0") 35 | 36 | DELTA_NO_ADD = 0xffffffff 37 | 38 | # From 39 | POLLIN = 1 40 | POLLOUT = 4 41 | 42 | # OMcache uses sys/syslog.h priority numbers 43 | LOG_ERR = 3 44 | LOG_WARNING = 4 45 | LOG_NOTICE = 5 46 | LOG_INFO = 6 47 | LOG_DEBUG = 7 48 | 49 | # Memcache binary protocol command opcodes 50 | CMD_GET = 0x00 51 | CMD_SET = 0x01 52 | CMD_ADD = 0x02 53 | CMD_REPLACE = 0x03 54 | CMD_DELETE = 0x04 55 | CMD_INCREMENT = 0x05 56 | CMD_DECREMENT = 0x06 57 | CMD_QUIT = 0x07 58 | CMD_FLUSH = 0x08 59 | CMD_GETQ = 0x09 60 | CMD_NOOP = 0x0a 61 | CMD_VERSION = 0x0b 62 | CMD_GETK = 0x0c 63 | CMD_GETKQ = 0x0d 64 | CMD_APPEND = 0x0e 65 | CMD_PREPEND = 0x0f 66 | CMD_STAT = 0x10 67 | CMD_SETQ = 0x11 68 | CMD_ADDQ = 0x12 69 | CMD_REPLACEQ = 0x13 70 | CMD_DELETEQ = 0x14 71 | CMD_INCREMENTQ = 0x15 72 | CMD_DECREMENTQ = 0x16 73 | CMD_QUITQ = 0x17 74 | CMD_FLUSHQ = 0x18 75 | CMD_APPENDQ = 0x19 76 | CMD_PREPENDQ = 0x1a 77 | CMD_TOUCH = 0x1c 78 | CMD_GAT = 0x1d 79 | CMD_GATQ = 0x1e 80 | CMD_GATK = 0x23 81 | CMD_GATKQ = 0x24 82 | 83 | 84 | class Error(Exception): 85 | """OMcache error""" 86 | 87 | 88 | class CommandError(Error): 89 | status = None 90 | 91 | def __init__(self, msg=None, status=None): 92 | super(CommandError, self).__init__(msg) 93 | if status is not None: 94 | self.status = status 95 | 96 | 97 | class NotFoundError(CommandError): 98 | status = _oc.OMCACHE_NOT_FOUND 99 | 100 | 101 | class KeyExistsError(CommandError): 102 | status = _oc.OMCACHE_KEY_EXISTS 103 | 104 | 105 | class TooLargeValueError(CommandError): 106 | status = _oc.OMCACHE_TOO_LARGE_VALUE 107 | 108 | 109 | class NotStoredError(CommandError): 110 | status = _oc.OMCACHE_NOT_STORED 111 | 112 | 113 | class DeltaBadValueError(CommandError): 114 | status = _oc.OMCACHE_DELTA_BAD_VALUE 115 | 116 | 117 | if version_info[0] >= 3: 118 | _select_errno = lambda e: e.errno 119 | _to_bytes = lambda s: s.encode("utf-8") if isinstance(s, str) else s 120 | def _to_string(s): 121 | msg = _ffi.string(s) 122 | if isinstance(msg, bytes): 123 | msg = msg.decode("utf-8") 124 | return msg 125 | else: 126 | _select_errno = lambda e: e[0] 127 | _to_bytes = lambda s: s.encode("utf-8") if isinstance(s, unicode) else s # pylint: disable=E0602 128 | _to_string = _ffi.string 129 | 130 | 131 | if socket.htonl(1) == 1: 132 | def _htobe64(v): 133 | return v 134 | else: 135 | def _htobe64(v): 136 | h = socket.htonl(v >> 32) 137 | l = socket.htonl(v & 0xffffffff) 138 | return (l << 32) | h 139 | 140 | 141 | OMcacheValue = namedtuple("OMcacheValue", ["status", "key", "value", "flags", "cas", "delta_value"]) 142 | 143 | 144 | def _omc_command(func, expected_values=1): 145 | @wraps(func) 146 | def omc_async_call(self, *args, **kwargs): 147 | func_name = kwargs.get("func_name", func.__name__) 148 | timeout = kwargs.pop("timeout", self.io_timeout) 149 | # `objs` holds references to CFFI objects which we need to hold for a while 150 | req, objs = func(self, *args, **kwargs) # pylint: disable=W0612 151 | if self.select == select_select: 152 | ret = _oc.omcache_command_status(self.omc, req, timeout) 153 | else: 154 | resps = self._omc_command_async(req, expected_values, timeout, func_name) 155 | ret = resps[0].status if resps else _oc.OMCACHE_FAIL 156 | return self._omc_check(ret, func_name) 157 | return omc_async_call 158 | 159 | 160 | class OMcache(object): 161 | def __init__(self, server_list, log=None, select=None): 162 | self.omc = _oc.omcache_init() 163 | self._log_cb = None 164 | self._log = None 165 | self.log = log 166 | self.select = select or select_select 167 | self._buffering = False 168 | self._conn_timeout = None 169 | self._reconn_timeout = None 170 | self._dead_timeout = None 171 | self.set_servers(server_list) 172 | self.io_timeout = 1000 173 | 174 | def __del__(self): 175 | self.free() 176 | 177 | def free(self): 178 | omc = getattr(self, "omc", None) 179 | if omc is not None: 180 | _oc.omcache_free(omc) 181 | self.omc = None 182 | 183 | @property 184 | def log(self): 185 | return self._log 186 | 187 | @log.setter 188 | def log(self, log): 189 | level = 0 190 | self._log = log 191 | if not log: 192 | self._log_cb = None 193 | _oc.omcache_set_log_callback(self.omc, level, _ffi.NULL, _ffi.NULL) 194 | return 195 | 196 | if hasattr(log, "getEffectiveLevel"): 197 | pyloglevel = log.getEffectiveLevel() 198 | if pyloglevel <= logging.DEBUG: 199 | level = LOG_DEBUG 200 | elif pyloglevel <= logging.INFO: 201 | level = LOG_INFO 202 | elif pyloglevel < logging.WARNING: 203 | # NOTICE is something between INFO and WARNING, not defined 204 | # in Python by default. 205 | level = LOG_NOTICE 206 | elif pyloglevel <= logging.WARNING: 207 | level = LOG_WARNING 208 | elif pyloglevel <= logging.ERROR: 209 | level = LOG_ERR 210 | else: 211 | # OMcache doesn't use anything more severe than LOG_ERR 212 | level = LOG_ERR - 1 213 | 214 | def _omc_log(context, level, msg): 215 | msg = _to_string(msg) 216 | if level <= LOG_ERR: 217 | log.error(msg) 218 | elif level == LOG_WARNING: 219 | log.warning(msg) 220 | elif level == LOG_NOTICE: 221 | # NOTE: python doesn't have NOTICE, but INFO is defined as 222 | # 20 and WARNING as 30 so let's use INFO + 1 for NOTICE 223 | log.log(logging.INFO + 1, msg) 224 | elif level == LOG_DEBUG: 225 | log.debug(msg) 226 | else: 227 | log.info(msg) 228 | self._log_cb = _ffi.callback("void(void*, int, const char *)", _omc_log) 229 | _oc.omcache_set_log_callback(self.omc, level, self._log_cb, _ffi.NULL) 230 | 231 | @staticmethod 232 | def _omc_check(ret, name, allowed=None): 233 | allowed = allowed if allowed is not None else [_oc.OMCACHE_BUFFERED] 234 | if ret == _oc.OMCACHE_OK or ret in allowed: 235 | return ret 236 | if ret == _oc.OMCACHE_NOT_FOUND: 237 | raise NotFoundError 238 | if ret == _oc.OMCACHE_KEY_EXISTS: 239 | raise KeyExistsError 240 | if ret == _oc.OMCACHE_TOO_LARGE_VALUE: 241 | raise TooLargeValueError 242 | if ret == _oc.OMCACHE_NOT_STORED: 243 | raise NotStoredError 244 | if ret == _oc.OMCACHE_DELTA_BAD_VALUE: 245 | raise DeltaBadValueError 246 | errstr = _to_string(_oc.omcache_strerror(ret)) 247 | raise CommandError("{0}: {1}".format(name, errstr), status=ret) 248 | 249 | def set_servers(self, server_list): 250 | if isinstance(server_list, (list, set, tuple)): 251 | server_list = ",".join(server_list) 252 | return _oc.omcache_set_servers(self.omc, _to_bytes(server_list)) 253 | 254 | def set_distribution_method(self, method): 255 | if method == "libmemcached_ketama": 256 | ms = _ffi.addressof(_oc.omcache_dist_libmemcached_ketama) 257 | elif method == "libmemcached_ketama_weighted": 258 | ms = _ffi.addressof(_oc.omcache_dist_libmemcached_ketama_weighted) 259 | elif method == "libmemcached_ketama_pre1010": 260 | ms = _ffi.addressof(_oc.omcache_dist_libmemcached_ketama_pre1010) 261 | else: 262 | raise Error("invalid distribution method {0!r}".format(method)) 263 | return _oc.omcache_set_distribution_method(self.omc, ms) 264 | 265 | @property 266 | def connect_timeout(self): 267 | return self._conn_timeout 268 | 269 | @connect_timeout.setter 270 | def connect_timeout(self, msec): 271 | self._conn_timeout = msec 272 | return _oc.omcache_set_connect_timeout(self.omc, msec) 273 | 274 | @property 275 | def reconnect_timeout(self): 276 | return self._reconn_timeout 277 | 278 | @reconnect_timeout.setter 279 | def reconnect_timeout(self, msec): 280 | self._reconn_timeout = msec 281 | return _oc.omcache_set_reconnect_timeout(self.omc, msec) 282 | 283 | @property 284 | def dead_timeout(self): 285 | return self._dead_timeout 286 | 287 | @dead_timeout.setter 288 | def dead_timeout(self, msec): 289 | self._dead_timeout = msec 290 | return _oc.omcache_set_dead_timeout(self.omc, msec) 291 | 292 | @property 293 | def buffering(self): 294 | return self._buffering 295 | 296 | @buffering.setter 297 | def buffering(self, enabled): 298 | self._buffering = True if enabled else False 299 | return _oc.omcache_set_buffering(self.omc, enabled) 300 | 301 | def reset_buffers(self): 302 | return _oc.omcache_reset_buffers(self.omc) 303 | 304 | def _omc_io(self, requests, request_count, values, value_count, timeout): 305 | nfdsp = _ffi.new("int *") 306 | polltimeoutp = _ffi.new("int *") 307 | polls = _oc.omcache_poll_fds(self.omc, nfdsp, polltimeoutp) 308 | rlist, wlist = [], [] 309 | for i in range(nfdsp[0]): 310 | if polls[i].events & POLLIN: 311 | rlist.append(polls[i].fd) 312 | if polls[i].events & POLLOUT: 313 | wlist.append(polls[i].fd) 314 | if rlist or wlist: 315 | if timeout <= 0: 316 | timeout = 0 317 | else: 318 | timeout = min(polltimeoutp[0], timeout) / 1000.0 319 | try: 320 | self.select(rlist, wlist, [], timeout) 321 | except select_error as ex: 322 | if _select_errno(ex) != errno.EINTR: 323 | raise 324 | ret = _oc.omcache_io(self.omc, requests, request_count, values, value_count, 0) 325 | self._omc_check(ret, "omcache_io", allowed=[_oc.OMCACHE_AGAIN]) 326 | if values == _ffi.NULL: 327 | yield OMcacheValue(ret, None, None, None, None, None) 328 | return 329 | for i in range(value_count[0]): 330 | key = _ffi.buffer(values[i].key, values[i].key_len)[:] 331 | value = _ffi.buffer(values[i].data, values[i].data_len)[:] 332 | yield OMcacheValue(values[i].status, key, value, values[i].flags, values[i].cas, values[i].delta_value) 333 | 334 | def _omc_command_async(self, requests, value_count, timeout, func_name): 335 | request_count = _ffi.new("size_t *") 336 | request_count[0] = len(requests) 337 | if value_count is None: 338 | value_count = len(requests) 339 | value_countp = _ffi.new("size_t *") 340 | values = _ffi.new("omcache_value_t[]", value_count) 341 | begin = time.time() 342 | results = [] 343 | ret = _oc.omcache_command(self.omc, requests, request_count, _ffi.NULL, _ffi.NULL, 0) 344 | self._omc_check(ret, func_name, allowed=[_oc.OMCACHE_AGAIN, _oc.OMCACHE_BUFFERED]) 345 | while request_count[0]: 346 | value_countp[0] = value_count 347 | if timeout == -1: 348 | time_left = 3600 349 | else: 350 | time_left = timeout - (time.time() - begin) * 1000 351 | if time_left < 0: 352 | break 353 | results.extend(self._omc_io(requests, request_count, values, value_countp, time_left)) 354 | return results 355 | 356 | def flush(self, timeout=-1): 357 | if self.select == select_select: 358 | ret = _oc.omcache_io(self.omc, _ffi.NULL, _ffi.NULL, _ffi.NULL, _ffi.NULL, timeout) 359 | else: 360 | ret = _oc.OMCACHE_AGAIN 361 | time_left = 1 362 | begin = time.time() 363 | while ret == _oc.OMCACHE_AGAIN and time_left > 0: 364 | time_left = 3600 if timeout == -1 else timeout - (time.time() - begin) * 1000 365 | resps = list(self._omc_io(_ffi.NULL, _ffi.NULL, _ffi.NULL, _ffi.NULL, time_left)) 366 | ret = resps[0].status 367 | return self._omc_check(ret, "flush") 368 | 369 | def _request(self, opcode, key=None, data=None, extra=None, cas=0, server_index=-1, objects=None, request=None): 370 | if objects is None: 371 | objects = [] 372 | requests = None 373 | if request is None: 374 | requests = _ffi.new("omcache_req_t[]", 1) 375 | request = requests[0] 376 | request.server_index = server_index 377 | request.header.opcode = opcode 378 | if cas: 379 | request.header.cas = _htobe64(cas) 380 | bodylen = 0 381 | if extra: 382 | objects.append(extra) 383 | bodylen = _ffi.sizeof(extra) 384 | request.extra = extra 385 | request.header.extlen = bodylen 386 | if key is not None: 387 | if not isinstance(key, _ffi.CData): 388 | key = _ffi.new("unsigned char[]", key) 389 | objects.append(key) 390 | bodylen += len(key) - 1 391 | request.key = key 392 | request.header.keylen = socket.htons(len(key) - 1) 393 | if data is not None: 394 | if not isinstance(data, _ffi.CData): 395 | data = _ffi.new("unsigned char[]", data) 396 | objects.append(data) 397 | bodylen += len(data) - 1 398 | request.data = data 399 | request.header.bodylen = socket.htonl(bodylen) 400 | return requests, objects 401 | 402 | @_omc_command 403 | def noop(self, server_index=0, timeout=None): 404 | return self._request(CMD_NOOP, server_index=server_index) 405 | 406 | def stat(self, command="", server_index=0, timeout=None): 407 | # `objs` holds references to CFFI objects which we need to hold for a while 408 | req, objs = self._request(CMD_STAT, key=_to_bytes(command), server_index=server_index) # pylint: disable=W0612 409 | timeout = timeout if timeout is not None else self.io_timeout 410 | resps = self._omc_command_async(req, 100, timeout, "stat") 411 | results = {} 412 | for resp in resps: 413 | self._omc_check(resp.status, "stat") 414 | if not resp.key and not resp.value: 415 | break 416 | results[resp.key] = resp.value 417 | return results 418 | 419 | @_omc_command 420 | def _omc_set(self, key, value, expiration, flags, cas, timeout, opcode, func_name): 421 | extra = _ffi.new("uint32_t[]", 2) 422 | extra[0] = socket.htonl(flags) 423 | extra[1] = socket.htonl(expiration) 424 | return self._request(opcode, key=_to_bytes(key), data=_to_bytes(value), extra=extra, cas=cas) 425 | 426 | def set(self, key, value, expiration=0, flags=0, cas=0, timeout=None): 427 | return self._omc_set(key, value, expiration, flags, cas, timeout, CMD_SET, "set") 428 | 429 | def add(self, key, value, expiration=0, flags=0, timeout=None): 430 | return self._omc_set(key, value, expiration, flags, 0, timeout, CMD_ADD, "add") 431 | 432 | def replace(self, key, value, expiration=0, flags=0, timeout=None): 433 | return self._omc_set(key, value, expiration, flags, 0, timeout, CMD_REPLACE, "replace") 434 | 435 | @_omc_command 436 | def append(self, key, value, cas=0, timeout=None): 437 | return self._request(CMD_APPEND, key=_to_bytes(key), data=_to_bytes(value), cas=cas) 438 | 439 | @_omc_command 440 | def prepend(self, key, value, cas=0, timeout=None): 441 | return self._request(CMD_PREPEND, key=_to_bytes(key), data=_to_bytes(value), cas=cas) 442 | 443 | @_omc_command 444 | def delete(self, key, timeout=None): 445 | return self._request(CMD_DELETE, key=_to_bytes(key)) 446 | 447 | @_omc_command 448 | def touch(self, key, expiration=0, timeout=None): 449 | extra = _ffi.new("uint32_t[]", 1) 450 | extra[0] = socket.htonl(expiration) 451 | return self._request(CMD_TOUCH, key=_to_bytes(key), extra=extra) 452 | 453 | def get(self, key, flags=False, cas=False, timeout=None): 454 | # `objs` holds references to CFFI objects which we need to hold for a while 455 | req, objs = self._request(CMD_GETK, _to_bytes(key)) # pylint: disable=W0612 456 | timeout = timeout if timeout is not None else self.io_timeout 457 | resps = self._omc_command_async(req, None, timeout, "get") 458 | ret = resps[0].status if resps else _oc.OMCACHE_NOT_FOUND 459 | self._omc_check(ret, "get") 460 | resp = resps[0] 461 | if not flags and not cas: 462 | return resp.value 463 | elif flags and cas: 464 | return (resp.value, resp.flags, resp.cas) 465 | elif flags: 466 | return (resp.value, resp.flags) 467 | elif cas: 468 | return (resp.value, resp.cas) 469 | 470 | def get_multi(self, keys, flags=False, cas=False, timeout=None): 471 | if not isinstance(keys, (list, tuple)): 472 | keys = list(keys) 473 | objects = [] 474 | requests = _ffi.new("omcache_req_t[]", len(keys)) 475 | for i in range(len(keys)): 476 | self._request(CMD_GETKQ, _to_bytes(keys[i]), request=requests[i], objects=objects) 477 | timeout = timeout if timeout is not None else self.io_timeout 478 | resps = self._omc_command_async(requests, None, timeout, "get_multi") 479 | results = {} 480 | for resp in resps: 481 | if resp.status != _oc.OMCACHE_OK: 482 | continue 483 | if flags and cas: 484 | results[resp.key] = (resp.value, resp.flags, resp.cas) 485 | elif flags: 486 | results[resp.key] = (resp.value, resp.flags) 487 | elif cas: 488 | results[resp.key] = (resp.value, resp.cas) 489 | else: 490 | results[resp.key] = resp.value 491 | return results 492 | 493 | def _omc_delta(self, key, delta, initial, expiration, timeout, func_name): 494 | # Delta operation definition in the protocol is a bit weird; if 495 | # 'expiration' is set to DELTA_NO_ADD (0xffffffff) the value will 496 | # not be created if it doesn't exist yet, but since we have a 497 | # 'initial' argument in the python api we'd really like to use it to 498 | # signal whether or not a value should be initialized. 499 | # 500 | # To do that we'll map expiration to DELTA_NO_ADD if initial is None 501 | # and expiration is empty and throw an error if initial is None but 502 | # expiration isn't empty. 503 | 504 | if delta < 0: 505 | opcode = CMD_DECREMENT 506 | delta = -delta 507 | else: 508 | opcode = CMD_INCREMENT 509 | if initial is None: 510 | if expiration: 511 | raise Error(func_name + " operation's initial must be set if expiration time is used") 512 | expiration = DELTA_NO_ADD 513 | initial = 0 514 | # CFFI doesn't support packed structs before version 0.8.2 so we create 515 | # an array of 5 x uint32_s instead of 2 x uint64_t + 1 x uint32_t 516 | extra = _ffi.new("uint32_t[]", 5) 517 | extra[0] = socket.htonl(delta >> 32) 518 | extra[1] = socket.htonl(delta & 0xffffffff) 519 | if initial: 520 | extra[2] = socket.htonl(initial >> 32) 521 | extra[3] = socket.htonl(initial & 0xffffffff) 522 | extra[4] = socket.htonl(expiration) 523 | # `objs` holds references to CFFI objects which we need to hold for a while 524 | req, objs = self._request(opcode, key=_to_bytes(key), extra=extra) # pylint: disable=W0612 525 | timeout = timeout if timeout is not None else self.io_timeout 526 | resps = self._omc_command_async(req, 1, timeout, func_name) 527 | ret = resps[0].status if resps else _oc.OMCACHE_FAIL 528 | self._omc_check(ret, func_name) 529 | return resps[0].delta_value 530 | 531 | def increment(self, key, delta=1, initial=None, expiration=0, timeout=None): 532 | return self._omc_delta(key, delta, initial, expiration, timeout, "increment") 533 | 534 | def decrement(self, key, delta=1, initial=None, expiration=0, timeout=None): 535 | return self._omc_delta(key, -delta, initial, expiration, timeout, "decrement") 536 | -------------------------------------------------------------------------------- /omcache.spec: -------------------------------------------------------------------------------- 1 | Name: libomcache 2 | Version: %{major_version} 3 | Release: %{minor_version}%{?dist} 4 | Summary: memcache client library 5 | Group: System Environment/Libraries 6 | URL: https://github.com/ohmu/omcache/ 7 | License: ASL 2.0 8 | Source0: omcache-rpm-src.tar.gz 9 | BuildRequires: check-devel, libasyncns-devel, memcached 10 | 11 | %description 12 | OMcache is a low level C library for accessing memcached servers. The goals 13 | of the OMcache project are stable API and ABI and 'easy' integration into 14 | complex applications and systems; OMcache specifically does not mask any 15 | signals or call any blocking functions. 16 | 17 | %package devel 18 | Summary: development files for omcache 19 | Group: Development/Libraries 20 | Requires: %{name} = %{version} 21 | 22 | %description devel 23 | Development libraries and headers for the OMcache memcache client library. 24 | 25 | %package -n python-omcache 26 | Summary: memcache client library for python 2.x 27 | Group: Development/Languages 28 | BuildArch: noarch 29 | BuildRequires: python-devel, pylint, pytest, python-cffi 30 | Requires: python-cffi, %{name} = %{version} 31 | 32 | %description -n python-omcache 33 | Python 2.x bindings for the OMcache memcache client library. 34 | 35 | %if %{?python3_sitelib:1}0 36 | %package -n python3-omcache 37 | Summary: memcache client library for python 3.x 38 | Group: Development/Languages 39 | BuildArch: noarch 40 | BuildRequires: python3-devel, python3-pylint, python3-pytest, python3-cffi 41 | Requires: python3-cffi, %{name} = %{version} 42 | 43 | %description -n python3-omcache 44 | Python 3.x bindings for the OMcache memcache client library. 45 | %endif 46 | 47 | %prep 48 | %setup -q -n omcache 49 | 50 | %build 51 | make 52 | 53 | %install 54 | rm -rf %{buildroot} 55 | make install DESTDIR=%{buildroot} \ 56 | PREFIX=%{_prefix} LIBDIR=%{_libdir} \ 57 | PYTHONDIRS="%{python2_sitelib} %{?python3_sitelib}" 58 | 59 | %check 60 | make check 61 | make check-pylint check-python PYTHON=python2 62 | %if %{?python3_sitelib:1}0 63 | make check-pylint check-python PYTHON=python3 64 | %endif 65 | 66 | %clean 67 | rm -rf %{buildroot} 68 | 69 | %files 70 | %defattr(-,root,root,-) 71 | %doc README.rst LICENSE NEWS 72 | %{_libdir}/libomcache.so.0 73 | 74 | %files devel 75 | %defattr(-,root,root,-) 76 | %{_libdir}/libomcache.so 77 | %{_includedir}/omcache.h 78 | %{_includedir}/omcache_cdef.h 79 | %{_includedir}/omcache_libmemcached.h 80 | 81 | %files -n python-omcache 82 | %defattr(-,root,root,-) 83 | %{python2_sitelib}/omcache* 84 | 85 | %if %{?python3_sitelib:1}0 86 | %files -n python3-omcache 87 | %defattr(-,root,root,-) 88 | %{python3_sitelib}/omcache* 89 | %{python3_sitelib}/__pycache__/omcache* 90 | %endif 91 | 92 | %changelog 93 | * Wed Jan 28 2015 Oskari Saarenmaa - 0.2.0-15-g34d33df 94 | - Don't package python3 bindings if python3_sitelib isn't defined 95 | - Run python tests and BuildRequire pytest and python-cffi 96 | - BuildRequire memcached, it's needed for make check 97 | 98 | * Mon Oct 13 2014 Oskari Saarenmaa - 0-unknown 99 | - Initial. 100 | -------------------------------------------------------------------------------- /omcache_libmemcached.h: -------------------------------------------------------------------------------- 1 | /* 2 | * omcache_libmemcached.h - a kludgy libmemcached API compatibility layer 3 | * 4 | * Copyright (c) 2013-2015, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | * NOTE: The functionality provided by this wrapper is limited and it is not 11 | * supported; it's provided to make it easier to prototype OMcache in simple 12 | * programs that use the libmemcached API. 13 | * 14 | */ 15 | 16 | #ifndef _OMCACHE_LIBMEMCACHED_H 17 | #define OMCACHE_LIBMEMCACHED_H 1 18 | 19 | #include "omcache.h" 20 | 21 | #define MEMCACHED_EXPIRATION_NOT_ADD OMCACHE_DELTA_NO_ADD 22 | #define MEMCACHED_SUCCESS OMCACHE_OK 23 | #define MEMCACHED_FAILURE OMCACHE_FAIL 24 | #define MEMCACHED_BUFFERED OMCACHE_BUFFERED 25 | #define MEMCACHED_NOTFOUND OMCACHE_NOT_FOUND 26 | #define MEMCACHED_END -1 27 | #define MEMCACHED_SOME_ERRORS -1 28 | #define LIBMEMCACHED_VERSION_HEX 0x01000003 29 | 30 | // how long should we wait for commands to complete? 31 | #ifndef MEMCACHED_COMMAND_TIMEOUT 32 | #define MEMCACHED_COMMAND_TIMEOUT -1 33 | #endif // !MEMCACHED_COMMAND_TIMEOUT 34 | 35 | #ifndef MEMCACHED_READ_TIMEOUT 36 | #define MEMCACHED_READ_TIMEOUT MEMCACHED_COMMAND_TIMEOUT 37 | #endif // !MEMCACHED_READ_TIMEOUT 38 | 39 | #ifndef MEMCACHED_WRITE_TIMEOUT 40 | #define MEMCACHED_WRITE_TIMEOUT MEMCACHED_COMMAND_TIMEOUT 41 | #endif // !MEMCACHED_WRITE_TIMEOUT 42 | 43 | // memcached functions usually uses signed char *s, omcache uses unsigned char *s 44 | #define omc_cc_to_cuc(v) (const unsigned char *) ({ const char *cc_ = (v); cc_; }) 45 | // we ignore some arguments and don't want to emit warnings about them 46 | #define omc_unused_var(v) ({ __typeof__(v) __attribute__((unused)) uu__ = (v); }) 47 | 48 | typedef omcache_t memcached_st; 49 | typedef char memcached_server_st; 50 | typedef void * memcached_stat_st; 51 | typedef int memcached_return; 52 | typedef int memcached_return_t; 53 | typedef omcache_server_info_t *memcached_server_instance_st; 54 | typedef memcached_return_t (*memcached_server_fn)(memcached_st *, memcached_server_instance_st, void *); 55 | 56 | typedef const char *memcached_behavior; 57 | typedef const char *memcached_behavior_t; 58 | typedef const char *memcached_hash; 59 | typedef const char *memcached_hash_t; 60 | typedef const char *memcached_server_distribution; 61 | typedef const char *memcached_server_distribution_t; 62 | 63 | #define memcached_create(mc) omcache_init() 64 | #define memcached_free(mc) omcache_free(mc) 65 | #define memcached_strerror(mc,rc) omcache_strerror(rc) 66 | #define memcached_flush_buffers(mc) omcache_io((mc), NULL, NULL, NULL, NULL, -1) 67 | #define memcached_flush(mc,expire) ({ \ 68 | int srvidx_, rc_ = OMCACHE_OK; \ 69 | for (srvidx_ = 0; rc_ == OMCACHE_OK; srvidx_ ++) \ 70 | rc_ = omcache_flush_all((mc), (expire), srvidx_, -1) ; \ 71 | (rc_ == OMCACHE_NO_SERVERS) ? OMCACHE_OK : rc_; }) 72 | #define memcached_increment(mc,key,key_len,offset,val) \ 73 | omcache_increment((mc), omc_cc_to_cuc(key), (key_len), (offset), 0, OMCACHE_DELTA_NO_ADD, (val), MEMCACHED_WRITE_TIMEOUT) 74 | #define memcached_increment_with_initial(mc,key,key_len,offset,initial,expire,val) \ 75 | omcache_increment((mc), omc_cc_to_cuc(key), (key_len), (offset), (initial), (expire), (val), MEMCACHED_WRITE_TIMEOUT) 76 | #define memcached_decrement(mc,key,key_len,offset,val) \ 77 | omcache_decrement((mc), omc_cc_to_cuc(key), (key_len), (offset), 0, OMCACHE_DELTA_NO_ADD, (val), MEMCACHED_WRITE_TIMEOUT) 78 | #define memcached_decrement_with_initial(mc,key,key_len,offset,initial,expire,val) \ 79 | omcache_decrement((mc), omc_cc_to_cuc(key), (key_len), (offset), (initial), (expire), (val), MEMCACHED_WRITE_TIMEOUT) 80 | #define memcached_add(mc,key,key_len,val,val_len,expire,flags) \ 81 | omcache_add((mc), omc_cc_to_cuc(key), (key_len), omc_cc_to_cuc(val), (val_len), (expire), (flags), MEMCACHED_WRITE_TIMEOUT) 82 | #define memcached_set(mc,key,key_len,val,val_len,expire,flags) \ 83 | omcache_set((mc), omc_cc_to_cuc(key), (key_len), omc_cc_to_cuc(val), (val_len), (expire), (flags), 0, MEMCACHED_WRITE_TIMEOUT) 84 | #define memcached_replace(mc,key,key_len,val,val_len,expire,flags) \ 85 | omcache_replace((mc), omc_cc_to_cuc(key), (key_len), omc_cc_to_cuc(val), (val_len), (expire), (flags), MEMCACHED_WRITE_TIMEOUT) 86 | #define memcached_touch(mc,key,key_len,expire) \ 87 | omcache_touch((mc), omc_cc_to_cuc(key), (key_len), (expire)) 88 | // NOTE: memcached protocol doesn't have expiration for delete 89 | #define memcached_delete(mc,key,key_len,expire) \ 90 | ({ omc_unused_var(expire); omcache_delete((mc), omc_cc_to_cuc(key), (key_len), MEMCACHED_WRITE_TIMEOUT); }) 91 | // NOTE: memcached protocol doesn't have expiration or flags for append and prepend 92 | #define memcached_append(mc,key,key_len,val,val_len,expire,flags) \ 93 | ({ omc_unused_var(expire); omc_unused_var(flags); \ 94 | omcache_append((mc), omc_cc_to_cuc(key), (key_len), omc_cc_to_cuc(val), (val_len), 0, MEMCACHED_WRITE_TIMEOUT); }) 95 | #define memcached_prepend(mc,key,key_len,val,val_len,expire,flags) \ 96 | ({ omc_unused_var(expire); omc_unused_var(flags); \ 97 | omcache_prepend((mc), omc_cc_to_cuc(key), (key_len), omc_cc_to_cuc(val), (val_len), 0, MEMCACHED_WRITE_TIMEOUT); }) 98 | 99 | #define memcached_get(mc,key,key_len,r_len,flags,rc) \ 100 | ({ const unsigned char *val_; \ 101 | *rc = omcache_get((mc), omc_cc_to_cuc(key), (key_len), &val_, (r_len), (flags), NULL, MEMCACHED_READ_TIMEOUT); \ 102 | memcpy(malloc(*(r_len)), val_, *(r_len)); }) 103 | 104 | #define memcached_servers_parse(s) strdup(s) 105 | #define memcached_server_push omcache_set_servers 106 | #define memcached_server_list_free(s) free(s) 107 | 108 | #define memcached_server_name(s) (s)->hostname 109 | #define memcached_server_port(s) (s)->port 110 | #define memcached_server_cursor(mc,cb_list,cb_ctx,cb_cnt) \ 111 | ({ int cbidx_, srvidx_ = 0, res_ = MEMCACHED_SUCCESS; \ 112 | omcache_server_info_t *srvnfo_ = NULL; \ 113 | while ((res_ == MEMCACHED_SUCCESS) && (srvnfo_ = omcache_server_info((mc), srvidx_))) { \ 114 | for (cbidx_ = 0; (res_ == MEMCACHED_SUCCESS) && (cbidx_ < (cb_cnt)); cbidx_ ++) { \ 115 | res_ = (cb_list)[cbidx_]((mc), srvnfo_, (cb_ctx)); \ 116 | } srvidx_ ++; \ 117 | omcache_server_info_free((mc), srvnfo_); } \ 118 | res_; }) 119 | 120 | // omcache can't implement memcached_stat_servername which doesn't use memcached_st 121 | // use omcache_stat_by_server_name() instead 122 | #define memcached_stat_servername(stat,keys,hostname,port) \ 123 | ({ *(stat) = NULL; MEMCACHED_SUCCESS; }) 124 | #define memcached_stat_get_keys(mc,stat,rc) \ 125 | ({ *(rc) = MEMCACHED_SUCCESS; NULL; }) 126 | #define memcached_stat_get_value(mc,stat,key,rc) \ 127 | ({ *(rc) = MEMCACHED_FAILURE; NULL; }) 128 | 129 | // various omcache_set_* apis need to be used instead of behaviors 130 | #define memcached_behavior_set(m,k,v) MEMCACHED_FAILURE 131 | 132 | // omcache_get_multi and omcache_io need to be called instead of these 133 | #define memcached_mget(mc,keys,key_lens,arr_len) MEMCACHED_FAILURE 134 | #define memcached_fetch(mc,key,key_len,val_len,flags,rc) ({ *rc = MEMCACHED_FAILURE; NULL; }) 135 | 136 | #endif // !_OMCACHE_LIBMEMCACHED_H 137 | -------------------------------------------------------------------------------- /omcache_priv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * OMcache - a memcached client library 3 | * 4 | * Copyright (c) 2013-2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #ifndef _OMCACHE_PRIV_H 13 | #define _OMCACHE_PRIV_H 1 14 | 15 | #include "omcache.h" 16 | #include "memcached_protocol_binary.h" 17 | #include "compat.h" 18 | 19 | #define MC_PORT "11211" 20 | 21 | #define omc_hidden __attribute__((visibility("hidden"))) 22 | 23 | typedef struct omc_hash_node_s 24 | { 25 | uint32_t key; 26 | void *val; 27 | struct omc_hash_node_s *next; 28 | } omc_hash_node_t; 29 | 30 | typedef struct omc_hash_table_s 31 | { 32 | uint32_t size; 33 | uint32_t count; 34 | void *not_found_val; 35 | omc_hash_node_t **buckets; 36 | omc_hash_node_t *freelist; 37 | omc_hash_node_t nodes[]; 38 | } omc_hash_table_t; 39 | 40 | omc_hidden omc_hash_table_t *omc_hash_table_init(omc_hash_table_t *hash, uint32_t size, void *not_found_val); 41 | omc_hidden void omc_hash_table_free(omc_hash_table_t *hash); 42 | omc_hidden void *omc_hash_table_find(omc_hash_table_t *hash, uint32_t key); 43 | omc_hidden int omc_hash_table_add(omc_hash_table_t *hash, uint32_t key, void *val); 44 | omc_hidden void *omc_hash_table_del(omc_hash_table_t *hash, uint32_t key); 45 | 46 | #define omc_int_hash_table_t omc_hash_table_t 47 | #define omc_int_hash_table_init(h,s) omc_hash_table_init((h), (s), (void *) (uintptr_t) -1) 48 | #define omc_int_hash_table_free(h) omc_hash_table_free(h) 49 | #define omc_int_hash_table_find(h,k) ((intptr_t) omc_hash_table_find((h), (k))) 50 | #define omc_int_hash_table_add(h,k,v) omc_hash_table_add((h), (k), (void *) (uintptr_t) (v)) 51 | #define omc_int_hash_table_del(h,k) omc_hash_table_del((h), (k)) 52 | 53 | omc_hidden void omc_hash_md5(const unsigned char *key, size_t key_len, unsigned char *buf); 54 | omc_hidden uint32_t omc_hash_jenkins_oat(const unsigned char *key, size_t key_len); 55 | 56 | #endif // !_OMCACHE_PRIV_H 57 | -------------------------------------------------------------------------------- /omcache_pylibmc.py: -------------------------------------------------------------------------------- 1 | # omcache_pylibmc.py - a kludgy pylbmc API compatibility layer 2 | # 3 | # Copyright (c) 2013-2014, Oskari Saarenmaa 4 | # All rights reserved. 5 | # 6 | # This file is under the Apache License, Version 2.0. 7 | # See the file `LICENSE` for details. 8 | # 9 | # NOTE: The functionality provided by this wrapper is limited and it is not 10 | # supported; it's provided to make it easier to prototype OMcache in simple 11 | # programs that use the pylibmc API. 12 | 13 | from sys import version_info 14 | import omcache 15 | import warnings 16 | 17 | 18 | MemcachedError = omcache.CommandError 19 | NotFound = omcache.NotFoundError 20 | 21 | PYLIBMC_FLAG_PICKLE = 0x01 # not supported 22 | PYLIBMC_FLAG_INT = 0x02 23 | PYLIBMC_FLAG_LONG = 0x04 24 | PYLIBMC_FLAG_ZLIB = 0x08 # not supported 25 | PYLIBMC_FLAG_BOOL = 0x10 26 | 27 | 28 | if version_info[0] >= 3: 29 | _i_types = int 30 | _u_type = str 31 | else: 32 | _i_types = (int, long) # pylint: disable=E0602 33 | _u_type = unicode # pylint: disable=E0602 34 | 35 | def _s_value(value): 36 | flags = 0 37 | if isinstance(value, bool): 38 | flags |= PYLIBMC_FLAG_BOOL 39 | value = str(value) 40 | elif isinstance(value, _i_types): 41 | flags |= PYLIBMC_FLAG_INT 42 | value = str(value) 43 | if isinstance(value, _u_type): 44 | value = value.encode("utf-8") 45 | elif not isinstance(value, bytes): 46 | raise ValueError("Can't store value of type {0!r}".format(type(value))) 47 | return value, flags 48 | 49 | 50 | class Client(omcache.OMcache): 51 | def __init__(self, servers, behaviors=None, binary=None, username=None, password=None): 52 | if username or password: 53 | raise omcache.Error("OMcache does not support authentication at the moment") 54 | if binary is False: 55 | warnings.warn("OMcache always uses binary protocol, ignoring binary=False") 56 | super(Client, self).__init__(servers) 57 | self._behaviors = {} 58 | if behaviors: 59 | self.behaviors = behaviors 60 | 61 | @property 62 | def behaviors(self): 63 | return self._behaviors 64 | 65 | @behaviors.setter 66 | def behaviors(self, behaviors): 67 | for k, v in behaviors.items(): 68 | if k in ("cas", "no_block", "remove_failed", "auto_eject", "failure_limit"): 69 | pass 70 | elif k == "dead_timeout": 71 | self.dead_timeout = v * 1000 # seconds in pylibmc 72 | elif k == "retry_timeout": 73 | self.reconnect_timeout = v * 1000 # seconds in pylibmc 74 | elif k == "connect_timeout": 75 | self.connect_timeout = v # milliseconds in pylibmc 76 | elif k in ("ketama", "ketama_weighted", "ketama_pre1010") and v: 77 | self.set_distribution_method("libmemcached_" + k) 78 | else: 79 | warnings.warn("OMcache does not support behavior {0!r}: {1!r}".format(k, v)) 80 | continue 81 | self._behaviors[k] = v 82 | 83 | incr = omcache.OMcache.increment 84 | decr = omcache.OMcache.decrement 85 | 86 | @staticmethod 87 | def _deserialize_value(value, flags): 88 | if flags & (PYLIBMC_FLAG_PICKLE | PYLIBMC_FLAG_ZLIB): 89 | warnings.warn("Ignoring cache value {0!r} with unsupported flags 0x{1:x}".format(value, flags)) 90 | return None 91 | elif flags & (PYLIBMC_FLAG_INT | PYLIBMC_FLAG_LONG): 92 | return int(value) 93 | elif flags & PYLIBMC_FLAG_BOOL: 94 | return bool(value) 95 | return value 96 | 97 | def get(self, key, cas=False): 98 | try: 99 | value, flags, casval = super(Client, self).get(key, cas=True, flags=True) 100 | except NotFound: 101 | return None 102 | value = self._deserialize_value(value, flags) 103 | if cas: 104 | return (value, casval) 105 | return value 106 | 107 | def gets(self, key): 108 | return self.get(key, cas=True) 109 | 110 | def get_multi(self, keys, key_prefix=None): 111 | if key_prefix: 112 | keys = ["{0}{1}".format(key_prefix, key) for key in keys] 113 | values = super(Client, self).get_multi(keys, flags=True) 114 | result = {} 115 | for key, (value, flags) in values.items(): 116 | if key_prefix: 117 | key = key[len(key_prefix):] 118 | result[key] = self._deserialize_value(value, flags) 119 | return result 120 | 121 | def set(self, key, value, time=0): 122 | value, flags = _s_value(value) 123 | super(Client, self).set(key, value, expiration=time, flags=flags) 124 | return True 125 | 126 | def add(self, key, value, time=0): 127 | value, flags = _s_value(value) 128 | try: 129 | super(Client, self).add(key, value, expiration=time, flags=flags) 130 | return True 131 | except omcache.KeyExistsError: 132 | return False 133 | 134 | def cas(self, key, value, cas, time=0): 135 | value, flags = _s_value(value) 136 | try: 137 | super(Client, self).set(key, value, expiration=time, cas=cas, flags=flags) 138 | return True 139 | except omcache.KeyExistsError: 140 | return False 141 | 142 | def replace(self, key, value, time=0): 143 | value, flags = _s_value(value) 144 | try: 145 | super(Client, self).replace(key, value, expiration=time, flags=flags) 146 | return True 147 | except omcache.NotFoundError: 148 | return False 149 | 150 | def append(self, key, value): 151 | value, _ = _s_value(value) # ignore flags 152 | try: 153 | super(Client, self).append(key, value) 154 | return True 155 | except omcache.NotStoredError: 156 | return False 157 | 158 | def prepend(self, key, value): 159 | value, _ = _s_value(value) # ignore flags 160 | try: 161 | super(Client, self).prepend(key, value) 162 | return True 163 | except omcache.NotStoredError: 164 | return False 165 | 166 | def set_multi(self, mapping, time=0, key_prefix=None): 167 | # pylibmc's set_multi returns a list of failed keys, but we don't 168 | # have such an operation at the moment without blocking or using 169 | # response callbacks 170 | # XXX: handle failed sets 171 | failed = [] 172 | for key, value in mapping.items(): 173 | try: 174 | prefixed_key = "{0}{1}".format(key_prefix or "", key) 175 | value, flags = _s_value(value) 176 | super(Client, self).set(prefixed_key, value, flags=flags, 177 | expiration=time, timeout=0) 178 | except omcache.CommandError: 179 | failed.append(key) 180 | return failed 181 | 182 | def delete(self, key): 183 | try: 184 | super(Client, self).delete(key) 185 | return True 186 | except omcache.NotFoundError: 187 | return False 188 | 189 | def delete_multi(self, keys, time=0, key_prefix=None): 190 | # pylibmc's delete_multi returns False if all keys weren't 191 | # successfully deleted (for example if they didn't exist at all), 192 | # but we don't have such an operation at the moment without blocking 193 | # or using response callbacks 194 | # XXX: handle failed deletes 195 | # NOTE: time argument is not supported by omcache 196 | success = True 197 | for key in keys: 198 | try: 199 | prefixed_key = "{0}{1}".format(key_prefix or "", key) 200 | super(Client, self).delete(prefixed_key, timeout=0) 201 | except omcache.CommandError: 202 | success = False 203 | return success 204 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | # -*- conf -*- 2 | [MESSAGES CONTROL] 3 | # I0011:143,0: : Locally disabling unused-variable (W0612) 4 | # W0212:145,19: _omc_command.omc_async_call: Access to a protected member _omc_check of a client class 5 | # W0221: 97,4: Client.get: Arguments number differs from overridden method 6 | # W0511:170,0: : XXX: handle failed sets 7 | # W0613:412,26: OMcache.delete: Unused argument 'timeout' 8 | disable=C,R,I0011,W0212,W0221,W0511,W0613 9 | 10 | [REPORTS] 11 | reports=no 12 | msg-template={msg_id}:{line:3d},{column}: {obj}: {msg} 13 | -------------------------------------------------------------------------------- /symbol.map: -------------------------------------------------------------------------------- 1 | OMCACHE_0.1 2 | { 3 | global: 4 | omcache_init; 5 | omcache_free; 6 | omcache_strerror; 7 | 8 | omcache_set_buffering; 9 | omcache_reset_buffers; 10 | 11 | omcache_set_connect_timeout; 12 | omcache_set_reconnect_timeout; 13 | omcache_set_dead_timeout; 14 | omcache_set_send_buffer_max_size; 15 | omcache_set_recv_buffer_max_size; 16 | omcache_set_response_callback; 17 | omcache_set_servers; 18 | 19 | omcache_set_distribution_method; 20 | omcache_dist_libmemcached_ketama; 21 | omcache_dist_libmemcached_ketama_weighted; 22 | omcache_dist_libmemcached_ketama_pre1010; 23 | 24 | omcache_set_log_callback; 25 | omcache_log_stderr; 26 | 27 | omcache_command; 28 | omcache_command_status; 29 | omcache_io; 30 | omcache_poll_fds; 31 | 32 | omcache_server_index_for_key; 33 | omcache_server_info; 34 | omcache_server_info_free; 35 | 36 | omcache_noop; 37 | omcache_stat; 38 | omcache_flush_all; 39 | omcache_set; 40 | omcache_add; 41 | omcache_replace; 42 | omcache_increment; 43 | omcache_decrement; 44 | omcache_delete; 45 | omcache_get; 46 | omcache_get_multi; 47 | 48 | local: *; 49 | }; 50 | 51 | OMCACHE_0.2 52 | { 53 | global: 54 | omcache_append; 55 | omcache_prepend; 56 | omcache_touch; 57 | omcache_gat; 58 | omcache_gat_multi; 59 | } OMCACHE_0.1; 60 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | include ../compat.mk 2 | 3 | TEST = test_omcache 4 | OBJS = test_omcache.o test_commands.o test_failures.o \ 5 | test_libmcd_compat.o test_misc.o test_servers.o 6 | 7 | WITH_CFLAGS += -I.. 8 | 9 | ifneq ($(WITH_LIBMEMCACHED),) 10 | WITH_CFLAGS += -DWITH_LIBMEMCACHED 11 | WITH_LIBS += -lmemcached 12 | endif 13 | 14 | all: $(TEST) 15 | 16 | check: $(TEST) ../libomcache.$(SO_EXT) 17 | LD_LIBRARY_PATH=.. $(CHECKER) ./$< 18 | 19 | $(TEST): $(OBJS) ../libomcache.a 20 | $(CC) $(LDFLAGS) $^ -o $@ -lcheck $(WITH_LIBS) 21 | 22 | ../libomcache.a: 23 | $(MAKE) -C .. 24 | 25 | clean: 26 | $(RM) $(OBJS) $(TEST) 27 | -------------------------------------------------------------------------------- /tests/python/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import random 4 | import subprocess 5 | import time 6 | import unittest 7 | 8 | 9 | logging.basicConfig(level=logging.INFO) 10 | 11 | 12 | class OMcacheCase(unittest.TestCase): 13 | a_memcacheds = [] 14 | c_memcacheds = [] 15 | m_mc_index = None 16 | log = None 17 | 18 | def setup_method(self, method): 19 | self.m_mc_index = 0 20 | self.log = logging.getLogger(method.__name__) 21 | 22 | @classmethod 23 | def teardown_class(cls): 24 | for proc in cls.a_memcacheds: 25 | proc.kill() 26 | 27 | @classmethod 28 | def start_memcached(cls, addr=None): 29 | memcached_path = os.getenv("MEMCACHED_PATH") or "/usr/bin/memcached" 30 | if not addr: 31 | addr = "127.0.0.1" 32 | port = random.randrange(30000,60000) 33 | proc = subprocess.Popen([memcached_path, "-vp", str(port), "-l", addr], 34 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 35 | time.sleep(0.1) 36 | assert proc.poll() is None 37 | cls.a_memcacheds.append(proc) 38 | if addr is None: 39 | cls.c_memcacheds.append(proc) 40 | return "{0}:{1}".format(addr, port) 41 | 42 | def get_memcached(self): 43 | try: 44 | return self.c_memcacheds[self.m_mc_index] 45 | except IndexError: 46 | return self.start_memcached() 47 | finally: 48 | self.m_mc_index += 1 49 | -------------------------------------------------------------------------------- /tests/python/test_omcache.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import omcache 3 | import random 4 | import select 5 | from pytest import raises # pylint: disable=E0611 6 | from time import sleep 7 | from . import OMcacheCase 8 | 9 | 10 | class TestOmcache(OMcacheCase): 11 | def test_internal_utils(self): 12 | err = select.error(errno.EINTR, "interrupted") 13 | assert omcache._select_errno(err) == errno.EINTR 14 | 15 | def test_set_servers(self): 16 | servers = [self.get_memcached(), self.get_memcached()] 17 | oc = omcache.OMcache(servers, self.log) 18 | oc.set_servers(servers + ["127.0.0.1:2", "127.0.0.1:113333", "127.0.0.1:113000"]) 19 | oc.set_servers(servers * 8) 20 | oc.set_servers(servers) 21 | 22 | def test_stat(self): 23 | servers = [self.get_memcached(), self.get_memcached()] 24 | oc = omcache.OMcache(servers, self.log) 25 | s1 = oc.stat("settings", 0) 26 | s2 = oc.stat("settings", 1) 27 | assert s1 != s2 28 | s1 = oc.stat("", 0) 29 | s2 = oc.stat("", 1) 30 | assert s1 != s2 31 | assert len(s1) > 30 32 | assert len(s2) > 30 33 | oc.noop(0) 34 | oc.noop(1) 35 | 36 | def test_incr_decr(self): 37 | oc = omcache.OMcache([self.get_memcached()], self.log) 38 | with raises(omcache.NotFoundError): 39 | oc.increment("test_incr_decr", 2, initial=None) 40 | assert oc.increment("test_incr_decr", 2, initial=0) == 0 41 | assert oc.increment("test_incr_decr", 2) == 2 42 | assert oc.increment("test_incr_decr", 2) == 4 43 | assert oc.increment("test_incr_decr", 2, initial=42) == 6 44 | assert oc.decrement("test_incr_decr", 5) == 1 45 | assert oc.decrement("test_incr_decr", 5) == 0 46 | oc.set("test_incr_decr", "567") 47 | assert oc.decrement("test_incr_decr", 5) == 562 48 | oc.set("test_incr_decr", "x567") 49 | with raises(omcache.DeltaBadValueError): 50 | oc.decrement("test_incr_decr", 5) 51 | oc.set("test_incr_decr", "100") 52 | assert oc.decrement("test_incr_decr", -5) == 105 53 | assert oc.increment("test_incr_decr", -5) == 100 54 | with raises(omcache.Error): 55 | oc.increment("test_incr_decr_e", 2, expiration=42, initial=None) 56 | 57 | def test_add_replace_set_delete(self): 58 | oc = omcache.OMcache([self.get_memcached(), self.get_memcached()], self.log) 59 | with raises(omcache.NotFoundError): 60 | oc.replace("test_arsd", "replaced") 61 | with raises(omcache.NotFoundError): 62 | oc.get("test_arsd") 63 | oc.add("test_arsd", "added") 64 | assert oc.get("test_arsd") == b"added" 65 | oc.set("test_arsd", "set") 66 | assert oc.get("test_arsd") == b"set" 67 | oc.replace("test_arsd", "replaced") 68 | assert oc.get("test_arsd") == b"replaced" 69 | with raises(omcache.KeyExistsError): 70 | oc.add("test_arsd", "foobar") 71 | assert oc.get("test_arsd") == b"replaced" 72 | oc.delete("test_arsd") 73 | with raises(omcache.NotFoundError): 74 | oc.delete("test_arsd") 75 | oc.set("test_arsd", "arsd", flags=531) 76 | res, flags, cas = oc.get("test_arsd", flags=True, cas=True) # pylint: disable=W0632 77 | assert res == b"arsd" 78 | assert flags == 531 79 | assert cas > 0 80 | res, flags = oc.get("test_arsd", flags=True) # pylint: disable=W0632 81 | assert flags == 531 82 | 83 | def test_cas(self): 84 | oc = omcache.OMcache([self.get_memcached()], self.log) 85 | with raises(omcache.NotFoundError): 86 | oc.set("test_cas", "xxx", cas=42424242) 87 | oc.set("test_cas", "xxx") 88 | with raises(omcache.KeyExistsError): 89 | oc.set("test_cas", "xxx", cas=42424242) 90 | res, cas1 = oc.get("test_cas", cas=True) # pylint: disable=W0632 91 | assert res == b"xxx" 92 | assert cas1 > 0 93 | oc.set("test_cas", "42", cas=cas1) 94 | with raises(omcache.KeyExistsError): 95 | oc.set("test_cas", "zzz", cas=cas1) 96 | res, cas2 = oc.get("test_cas", cas=True) # pylint: disable=W0632 97 | assert res == b"42" 98 | assert cas2 > 0 99 | assert cas2 != cas1 100 | oc.increment("test_cas", 8) 101 | with raises(omcache.KeyExistsError): 102 | oc.set("test_cas", "zzz", cas=cas2) 103 | res, cas3 = oc.get("test_cas", cas=True) # pylint: disable=W0632 104 | assert res == b"50" 105 | assert cas3 > 0 106 | assert cas3 != cas2 107 | 108 | def test_touch(self): 109 | oc = omcache.OMcache([self.get_memcached()], self.log) 110 | oc.set("test_touch", "qwerty", expiration=2) 111 | assert oc.get("test_touch") == b"qwerty" 112 | sleep(2) 113 | with raises(omcache.NotFoundError): 114 | oc.get("test_touch") 115 | oc.set("test_touch", "qwerty", expiration=1) 116 | assert oc.get("test_touch") == b"qwerty" 117 | oc.touch("test_touch", expiration=3) 118 | sleep(2) 119 | assert oc.get("test_touch") == b"qwerty" 120 | 121 | def test_append_prepend(self): 122 | oc = omcache.OMcache([self.get_memcached()], self.log) 123 | oc.set("test_ap", "asdf") 124 | assert oc.get("test_ap") == b"asdf" 125 | oc.append("test_ap", "zxcvb") 126 | assert oc.get("test_ap") == b"asdfzxcvb" 127 | oc.prepend("test_ap", "qwerty") 128 | assert oc.get("test_ap") == b"qwertyasdfzxcvb" 129 | 130 | def test_multi(self): 131 | oc = omcache.OMcache([self.get_memcached(), self.get_memcached()], self.log) 132 | item_count = 123 133 | val = str(random.random()).encode("utf-8") 134 | for i in range(item_count): 135 | oc.set("test_multi_{0}".format(i * 2), val, flags=i) 136 | keys = ["test_multi_{0}".format(i) for i in range(item_count * 2)] 137 | random.shuffle(keys) 138 | results = oc.get_multi(keys) 139 | assert len(results) == item_count 140 | for i in range(item_count): 141 | assert results["test_multi_{0}".format(i * 2).encode("utf-8")] == val 142 | # test with flags and cas 143 | results = oc.get_multi(keys, flags=True) 144 | assert len(results) == item_count 145 | for i in range(item_count): 146 | assert results["test_multi_{0}".format(i * 2).encode("utf-8")] == (val, i) 147 | results = oc.get_multi(keys, cas=True) 148 | assert len(results) == item_count 149 | # count the number of distinct cas values, we can't just compare 150 | # them to the previous entry as we're using two memcache servers 151 | # which may use the same cas values 152 | casses = set() 153 | for i in range(item_count): 154 | res, cas = results["test_multi_{0}".format(i * 2).encode("utf-8")] 155 | assert res == val 156 | casses.add(cas) 157 | assert len(casses) > item_count / 3 158 | results = oc.get_multi(keys, cas=True, flags=True) 159 | assert len(results) == item_count 160 | casses = set() 161 | for i in range(item_count): 162 | res, flags, cas = results["test_multi_{0}".format(i * 2).encode("utf-8")] 163 | assert res == val 164 | assert flags == i 165 | casses.add(cas) 166 | assert len(casses) > item_count / 3 167 | 168 | def test_dist_methods(self): 169 | # just make sure the different distribution methods distribute keys, well, differently 170 | mc1 = self.get_memcached() 171 | mc2 = self.get_memcached() 172 | oc = omcache.OMcache([mc1, mc2], self.log) 173 | item_count = 123 174 | for i in range(item_count): 175 | oc.set("test_dist_{0}".format(i), "orig_dist") 176 | oc.flush() 177 | oc.set_distribution_method("libmemcached_ketama") 178 | for i in range(item_count): 179 | oc.set("test_dist_{0}".format(i), "ketama") 180 | oc.flush() 181 | oc.set_distribution_method("libmemcached_ketama_weighted") 182 | for i in range(item_count): 183 | oc.set("test_dist_{0}".format(i), "ketama_weighted") 184 | oc.flush() 185 | oc.set_distribution_method("libmemcached_ketama_pre1010") 186 | for i in range(item_count): 187 | oc.set("test_dist_{0}".format(i), "ketama_pre1010") 188 | oc.flush() 189 | with raises(omcache.Error): 190 | oc.set_distribution_method("xxx") 191 | keys = ["test_dist_{0}".format(i) for i in range(item_count)] 192 | oc.set_servers([mc1]) 193 | results = list(oc.get_multi(keys).values()) 194 | oc.set_servers([mc2]) 195 | results.extend(oc.get_multi(keys).values()) 196 | counts = {} 197 | for value in results: 198 | if value not in counts: 199 | counts[value] = 0 200 | counts[value] += 1 201 | assert set([b"ketama", b"ketama_weighted", b"ketama_pre1010"]).issuperset(counts) 202 | assert counts[b"ketama"] >= item_count / 10 203 | assert counts[b"ketama_weighted"] >= item_count / 10 204 | assert counts[b"ketama_pre1010"] >= item_count / 10 205 | -------------------------------------------------------------------------------- /tests/test_commands.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include 13 | #include "test_omcache.h" 14 | 15 | #define TIMEOUT 2000 16 | 17 | START_TEST(test_noop) 18 | { 19 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 20 | 21 | ck_omcache_ok(omcache_noop(oc, 0, TIMEOUT)); 22 | ck_omcache_ok(omcache_noop(oc, 1, TIMEOUT)); 23 | ck_omcache(omcache_noop(oc, 2, TIMEOUT), OMCACHE_NO_SERVERS); 24 | 25 | omcache_free(oc); 26 | } 27 | END_TEST 28 | 29 | START_TEST(test_stats) 30 | { 31 | omcache_t *oc = ot_init_omcache(3, LOG_INFO); 32 | omcache_value_t vals[100]; 33 | 34 | for (int i = 0; i < 3; i ++) 35 | { 36 | size_t val_count = sizeof(vals) / sizeof(vals[0]); 37 | ck_omcache_ok(omcache_stat(oc, NULL, vals, &val_count, i, TIMEOUT)); 38 | ck_assert_uint_ge(val_count, 10); 39 | } 40 | 41 | omcache_free(oc); 42 | } 43 | END_TEST 44 | 45 | START_TEST(test_flush_all) 46 | { 47 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 48 | const unsigned char key[] = "test_flush_all"; 49 | size_t key_len = sizeof(key) - 1; 50 | const unsigned char *get_val; 51 | size_t get_val_len; 52 | 53 | ck_omcache_ok(omcache_flush_all(oc, 0, 0, TIMEOUT)); 54 | ck_omcache_ok(omcache_flush_all(oc, 0, 1, TIMEOUT)); 55 | 56 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "bar", 3, 0, 42, 0, TIMEOUT)); 57 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &get_val_len, NULL, NULL, TIMEOUT)); 58 | ck_assert_uint_eq(get_val_len, 3); 59 | 60 | ck_omcache_ok(omcache_flush_all(oc, 0, 0, TIMEOUT)); 61 | ck_omcache_ok(omcache_flush_all(oc, 0, 1, TIMEOUT)); 62 | 63 | ck_omcache(omcache_get(oc, key, key_len, &get_val, &get_val_len, NULL, NULL, TIMEOUT), OMCACHE_NOT_FOUND); 64 | 65 | omcache_free(oc); 66 | } 67 | END_TEST 68 | 69 | START_TEST(test_set_get_delete) 70 | { 71 | unsigned char *val; 72 | const unsigned char key[] = "test_set_get_delete"; 73 | size_t key_len = sizeof(key) - 1; 74 | const unsigned char *get_val; 75 | size_t val_len, get_val_len; 76 | uint32_t flags; 77 | uint64_t cas; 78 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 79 | 80 | ck_omcache(omcache_get(oc, key, key_len, NULL, NULL, NULL, NULL, TIMEOUT), OMCACHE_NOT_FOUND); 81 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "bar", 3, 0, 42, 0, TIMEOUT)); 82 | 83 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, &flags, &cas, TIMEOUT)); 84 | ck_assert_uint_eq(val_len, 3); 85 | ck_assert_int_eq(memcmp(get_val, "bar", 3), 0); 86 | ck_assert_uint_eq(flags, 42); 87 | ck_assert(cas != 0); 88 | 89 | ck_omcache_ok(omcache_delete(oc, key, key_len, TIMEOUT)); 90 | ck_omcache(omcache_delete(oc, key, key_len, TIMEOUT), OMCACHE_NOT_FOUND); 91 | ck_omcache(omcache_get(oc, key, key_len, NULL, NULL, NULL, NULL, TIMEOUT), OMCACHE_NOT_FOUND); 92 | 93 | // memcached allows 1mb values by default 94 | val_len = 2048 * 1024; 95 | val = malloc(val_len); 96 | memset(val, 'O', val_len); 97 | // try with a too small send buffer 98 | omcache_set_send_buffer_max_size(oc, 5000); 99 | ck_omcache(omcache_set(oc, key, key_len, val, val_len, 0, 0, 0, TIMEOUT), OMCACHE_BUFFER_FULL); 100 | // make buffer larger, but try to send more than MC will accept 101 | omcache_set_send_buffer_max_size(oc, 5000000); 102 | // memcached allows 1mb values by default 103 | ck_omcache(omcache_set(oc, key, key_len, val, val_len, 0, 0, 0, TIMEOUT), OMCACHE_TOO_LARGE_VALUE); 104 | val_len = 1000 * 1000; 105 | ck_omcache_ok(omcache_set(oc, key, key_len, val, val_len, 0, 0, 0, TIMEOUT)); 106 | 107 | omcache_set_recv_buffer_max_size(oc, 1000); 108 | ck_omcache(omcache_get(oc, key, key_len, &get_val, &get_val_len, NULL, NULL, TIMEOUT), OMCACHE_BUFFER_FULL); 109 | omcache_set_recv_buffer_max_size(oc, 2000000); 110 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &get_val_len, NULL, NULL, TIMEOUT)); 111 | ck_assert_uint_eq(get_val_len, val_len); 112 | ck_assert_int_eq(memcmp(get_val, val, val_len), 0); 113 | free(val); 114 | 115 | omcache_free(oc); 116 | } 117 | END_TEST 118 | 119 | START_TEST(test_cas_and_flags) 120 | { 121 | const unsigned char key[] = "test_cas_and_flags"; 122 | size_t key_len = sizeof(key) - 1; 123 | const unsigned char *get_val; 124 | size_t val_len; 125 | uint32_t flags; 126 | uint64_t cas, cas2; 127 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 128 | 129 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "bar", 3, 0, 42, 0, TIMEOUT)); 130 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, &flags, &cas, TIMEOUT)); 131 | ck_assert_uint_eq(val_len, 3); 132 | ck_assert_int_eq(memcmp(get_val, "bar", 3), 0); 133 | ck_assert_uint_eq(flags, 42); 134 | ck_assert(cas != 0); 135 | 136 | ck_omcache(omcache_set(oc, key, key_len, (cuc *) "baz", 3, 0, 42, 0xdeadbeef, TIMEOUT), OMCACHE_KEY_EXISTS); 137 | ck_omcache(omcache_set(oc, key, key_len, (cuc *) "baz", 3, 0, 42, cas, TIMEOUT), OMCACHE_OK); 138 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, &flags, &cas2, TIMEOUT)); 139 | ck_assert_uint_eq(val_len, 3); 140 | ck_assert_uint_eq(flags, 42); 141 | ck_assert_uint_ne(cas, cas2); 142 | 143 | omcache_free(oc); 144 | } 145 | END_TEST 146 | 147 | START_TEST(test_add_and_replace) 148 | { 149 | const unsigned char key[] = "test_add_and_replace"; 150 | size_t key_len = sizeof(key) - 1; 151 | const unsigned char *get_val; 152 | size_t val_len; 153 | uint32_t flags; 154 | uint64_t cas; 155 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 156 | 157 | ck_omcache(omcache_replace(oc, key, key_len, (cuc *) "zxcv", 4, 0, 99, TIMEOUT), OMCACHE_NOT_FOUND); 158 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 0, 42, 0, TIMEOUT)); 159 | ck_omcache_ok(omcache_replace(oc, key, key_len, (cuc *) "bar", 3, 0, 99, TIMEOUT)); 160 | ck_omcache(omcache_add(oc, key, key_len, (cuc *) "zxcv", 4, 0, 99, TIMEOUT), OMCACHE_KEY_EXISTS); 161 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, &flags, &cas, TIMEOUT)); 162 | ck_assert_uint_eq(val_len, 3); 163 | ck_assert_int_eq(memcmp(get_val, "bar", 3), 0); 164 | ck_assert_uint_eq(flags, 99); 165 | ck_assert(cas != 0); 166 | 167 | omcache_free(oc); 168 | } 169 | END_TEST 170 | 171 | START_TEST(test_append_and_prepend) 172 | { 173 | const unsigned char key[] = "test_append_and_prepend"; 174 | size_t key_len = sizeof(key) - 1; 175 | const unsigned char *get_val; 176 | size_t val_len; 177 | uint64_t cas; 178 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 179 | 180 | ck_omcache(omcache_append(oc, key, key_len, (cuc *) "zxcv", 4, 0, TIMEOUT), OMCACHE_NOT_STORED); 181 | ck_omcache(omcache_prepend(oc, key, key_len, (cuc *) "zxcv", 4, 0, TIMEOUT), OMCACHE_NOT_STORED); 182 | 183 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 0, 42, 0, TIMEOUT)); 184 | ck_omcache_ok(omcache_append(oc, key, key_len, (cuc *) "!!", 2, 0, TIMEOUT)); 185 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, NULL, &cas, TIMEOUT)); 186 | ck_assert_uint_eq(val_len, 6); 187 | ck_assert_int_eq(memcmp(get_val, "asdf!!", 6), 0); 188 | 189 | ck_omcache(omcache_prepend(oc, key, key_len, (cuc *) "QWE", 3, 1, TIMEOUT), OMCACHE_KEY_EXISTS); 190 | ck_omcache_ok(omcache_prepend(oc, key, key_len, (cuc *) "QWE", 3, cas, TIMEOUT)); 191 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, NULL, &cas, TIMEOUT)); 192 | ck_assert_uint_eq(val_len, 9); 193 | ck_assert_int_eq(memcmp(get_val, "QWEasdf!!", 9), 0); 194 | 195 | omcache_free(oc); 196 | } 197 | END_TEST 198 | 199 | START_TEST(test_touch) 200 | { 201 | // touch is a new command in 1.4.8 202 | if (strcmp(ot_memcached_version(), "1.4.8") < 0) 203 | return; 204 | 205 | const unsigned char key[] = "test_touch"; 206 | size_t key_len = sizeof(key) - 1; 207 | const unsigned char *get_val; 208 | size_t val_len; 209 | uint64_t cas; 210 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 211 | 212 | ck_omcache(omcache_touch(oc, key, key_len, 4, TIMEOUT), OMCACHE_NOT_FOUND); 213 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 1, 0, 0, TIMEOUT)); 214 | usleep(1100000); 215 | // touch should fail, the value alreayd expired 216 | ck_omcache(omcache_touch(oc, key, key_len, 4, TIMEOUT), OMCACHE_NOT_FOUND); 217 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 1, 0, 0, TIMEOUT)); 218 | ck_omcache_ok(omcache_touch(oc, key, key_len, 10, TIMEOUT)); 219 | usleep(1500000); 220 | // value should've been extended by touch 221 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, NULL, &cas, TIMEOUT)); 222 | ck_assert_uint_eq(val_len, 4); 223 | ck_assert_int_eq(memcmp(get_val, "asdf", 4), 0); 224 | 225 | omcache_free(oc); 226 | } 227 | END_TEST 228 | 229 | START_TEST(test_gat) 230 | { 231 | // gat is a new command in 1.4.8 232 | if (strcmp(ot_memcached_version(), "1.4.8") < 0) 233 | return; 234 | 235 | const unsigned char key[] = "test_gat"; 236 | size_t key_len = sizeof(key) - 1; 237 | const unsigned char *get_val; 238 | size_t val_len; 239 | uint64_t cas; 240 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 241 | 242 | // set with 1 second timeout, sleep and try gat, it should fail as the value already expired 243 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 1, 0, 0, TIMEOUT)); 244 | usleep(1100000); 245 | ck_omcache(omcache_gat(oc, key, key_len, &get_val, &val_len, 4, NULL, &cas, TIMEOUT), OMCACHE_NOT_FOUND); 246 | 247 | // set with 1 second timeout, gat immediately, it should work as the value has not expired yet 248 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 1, 0, 0, TIMEOUT)); 249 | ck_omcache_ok(omcache_gat(oc, key, key_len, &get_val, &val_len, 3, NULL, &cas, TIMEOUT)); 250 | ck_assert_uint_eq(val_len, 4); 251 | ck_assert_int_eq(memcmp(get_val, "asdf", 4), 0); 252 | // now sleep and check that the value is still there (gat should've extended its validity) 253 | usleep(2000000); 254 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, NULL, &cas, TIMEOUT)); 255 | ck_assert_uint_eq(val_len, 4); 256 | ck_assert_int_eq(memcmp(get_val, "asdf", 4), 0); 257 | // NOTE: touch expiration time setting is broken in memcached <1.4.13-16-g045da59 258 | if (strcmp(ot_memcached_version(), "1.4.13") > 0) 259 | { 260 | // sleep some more, the value should have expired after this 261 | usleep(1000000); 262 | ck_omcache(omcache_gat(oc, key, key_len, &get_val, &val_len, 4, NULL, &cas, TIMEOUT), OMCACHE_NOT_FOUND); 263 | } 264 | 265 | omcache_free(oc); 266 | } 267 | END_TEST 268 | 269 | START_TEST(test_increment_and_decrement) 270 | { 271 | const unsigned char key[] = "test_increment_and_decrement"; 272 | size_t key_len = sizeof(key) - 1; 273 | const unsigned char *get_val; 274 | size_t val_len; 275 | uint64_t val_uint = 0; 276 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 277 | 278 | ck_omcache_ok(omcache_set(oc, key, key_len, (cuc *) "asdf", 4, 0, 42, 0, TIMEOUT)); 279 | ck_omcache(omcache_increment(oc, key, key_len, 12, 0, 0, &val_uint, TIMEOUT), OMCACHE_DELTA_BAD_VALUE); 280 | ck_assert_uint_eq(val_uint, 0); 281 | 282 | ck_omcache_ok(omcache_delete(oc, key, key_len, TIMEOUT)); 283 | ck_omcache_ok(omcache_increment(oc, key, key_len, 12, 3, 0, &val_uint, TIMEOUT)); 284 | ck_assert_uint_eq(val_uint, 3); 285 | ck_omcache_ok(omcache_increment(oc, key, key_len, 12, 0, 0, &val_uint, TIMEOUT)); 286 | ck_assert_uint_eq(val_uint, 15); 287 | ck_omcache_ok(omcache_get(oc, key, key_len, &get_val, &val_len, NULL, NULL, TIMEOUT)); 288 | ck_assert_uint_eq(val_len, 2); 289 | ck_assert_int_eq(memcmp(get_val, "15", 2), 0); 290 | ck_omcache_ok(omcache_decrement(oc, key, key_len, 1000, 3, 0, &val_uint, TIMEOUT)); 291 | ck_assert_uint_eq(val_uint, 0); 292 | ck_omcache(omcache_increment(oc, key + 1, key_len - 1, 1000, 1000, OMCACHE_DELTA_NO_ADD, &val_uint, TIMEOUT), 293 | OMCACHE_NOT_FOUND); 294 | ck_omcache(omcache_decrement(oc, key + 1, key_len - 1, 1000, 1000, OMCACHE_DELTA_NO_ADD, &val_uint, TIMEOUT), 295 | OMCACHE_NOT_FOUND); 296 | ck_assert_uint_eq(val_uint, 0); 297 | ck_omcache_ok(omcache_decrement(oc, key + 1, key_len - 1, 1000, 999, 0, &val_uint, TIMEOUT)); 298 | ck_assert_uint_eq(val_uint, 999); 299 | ck_omcache_ok(omcache_decrement(oc, key + 1, key_len - 1, 10, 999, 0, &val_uint, TIMEOUT)); 300 | ck_assert_uint_eq(val_uint, 989); 301 | ck_omcache_ok(omcache_increment(oc, key + 1, key_len - 1, 20, 999, 0, &val_uint, TIMEOUT)); 302 | ck_assert_uint_eq(val_uint, 1009); 303 | 304 | omcache_free(oc); 305 | } 306 | END_TEST 307 | 308 | START_TEST(test_req_id_wraparound) 309 | { 310 | // NOTE: omcache_t is opaque, but we need to mangle req_id for this test 311 | // if omcache_s layout changes this test will break in interesting ways. 312 | struct omcache_s_TEST 313 | { 314 | int64_t init_msec; 315 | uint32_t req_id; 316 | }; 317 | char *keys[1000]; 318 | size_t key_lens[1000]; 319 | omcache_t *oc = ot_init_omcache(2, LOG_INFO); 320 | struct omcache_s_TEST *oc_s = (struct omcache_s_TEST *) oc; 321 | ck_omcache_ok(omcache_set_buffering(oc, true)); 322 | for (int i = 0; i < 1000; i ++) 323 | { 324 | key_lens[i] = asprintf(&keys[i], "test_req_id_wraparound_%d", i); 325 | ck_omcache(OMCACHE_BUFFERED, 326 | omcache_set(oc, (cuc *) keys[i], key_lens[i], (cuc *) keys[i], key_lens[i], 0, 0, 0, 0)); 327 | } 328 | ck_omcache_ok(omcache_set_buffering(oc, false)); 329 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 330 | 331 | // set req_id to 100 below maximum and issue a multiget for 1000 entries which should force a wraparound 332 | uint32_t pre_wrap_req_id = oc_s->req_id; 333 | oc_s->req_id = UINT32_MAX - 100; 334 | 335 | omcache_value_t values[1000]; 336 | size_t value_count = 1000, values_found = 0; 337 | omcache_req_t reqs[1000]; 338 | size_t req_count = 1000; 339 | ck_omcache_ok_or_again(omcache_get_multi(oc, (cuc **) keys, key_lens, 1000, reqs, &req_count, values, &value_count, 5000)); 340 | values_found = value_count; 341 | while (req_count > 0) 342 | { 343 | value_count = 1000; 344 | ck_omcache_ok_or_again(omcache_io(oc, reqs, &req_count, values, &value_count, 5000)); 345 | values_found += value_count; 346 | } 347 | ck_assert_int_eq(values_found, 1000); 348 | ck_assert_uint_le(oc_s->req_id, pre_wrap_req_id); 349 | for (int i = 0; i < 1000; i ++) 350 | free(keys[i]); 351 | 352 | omcache_free(oc); 353 | } 354 | END_TEST 355 | 356 | START_TEST(test_buffering) 357 | { 358 | char *keys[1000]; 359 | size_t key_lens[1000]; 360 | omcache_t *oc = ot_init_omcache(3, LOG_INFO); 361 | ck_omcache_ok(omcache_set_buffering(oc, true)); 362 | for (int i = 0; i < 1000; i += 2) 363 | { 364 | key_lens[i] = asprintf(&keys[i], "test_buffering_%d", i); 365 | ck_omcache(OMCACHE_BUFFERED, 366 | omcache_set(oc, (cuc *) keys[i], key_lens[i], (cuc *) keys[i], key_lens[i], 0, 0, 0, 0)); 367 | } 368 | ck_omcache_ok(omcache_reset_buffers(oc)); 369 | for (int i = 1; i < 1000; i += 2) 370 | { 371 | key_lens[i] = asprintf(&keys[i], "test_buffering_%d", i); 372 | ck_omcache(OMCACHE_BUFFERED, 373 | omcache_set(oc, (cuc *) keys[i], key_lens[i], (cuc *) keys[i], key_lens[i], 0, 0, 0, 0)); 374 | } 375 | ck_omcache_ok(omcache_set_buffering(oc, false)); 376 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 377 | 378 | // no even keys should be set 379 | omcache_value_t values[1000]; 380 | size_t value_count = 1000, values_found = 0; 381 | omcache_req_t reqs[1000]; 382 | size_t req_count = 1000; 383 | ck_omcache_ok_or_again(omcache_get_multi(oc, (cuc **) keys, key_lens, 1000, reqs, &req_count, values, &value_count, 5000)); 384 | values_found = value_count; 385 | while (req_count > 0) 386 | { 387 | value_count = 1000; 388 | ck_omcache_ok_or_again(omcache_io(oc, reqs, &req_count, values, &value_count, 5000)); 389 | values_found += value_count; 390 | // check that the last character of key is odd (ord("0") % 2 == 0) 391 | for (size_t i = 0; i < value_count; i ++) 392 | ck_assert_int_eq(1, values[i].key[values[i].key_len - 1] % 2); 393 | } 394 | ck_assert_int_eq(values_found, 500); 395 | 396 | // test trying to fetch non-matching request range 397 | // omcache_get_multi fails with a too low value count 398 | req_count = 999; 399 | ck_omcache(OMCACHE_INVALID, 400 | omcache_io(oc, reqs, &req_count, NULL, NULL, 5000)); 401 | // but omcache_stat doesn't really know how many values we will receive, 402 | // so it'll accept the request but silently drop some messages 403 | value_count = 1; 404 | ck_omcache_ok(omcache_stat(oc, "", values, &value_count, 0, 5000)); 405 | for (int i = 0; i < 1000; i ++) 406 | free(keys[i]); 407 | 408 | omcache_free(oc); 409 | } 410 | END_TEST 411 | 412 | static void test_response_callback_cb(omcache_t *mc omc_attribute_unused, 413 | omcache_value_t *result, void *context) 414 | { 415 | size_t *values_found_p = (size_t *) context; 416 | // NOTE: don't compare keys, memcached's GATKQ handling doesn't return 417 | // keys currently https://github.com/memcached/memcached/pull/85 418 | if (result->status == OMCACHE_OK && 419 | result->data_len >= sizeof("test_response_callback_") && 420 | memcmp(result->data, "test_response_callback_", sizeof("test_response_callback_") - 1) == 0) 421 | { 422 | *values_found_p = (*values_found_p) + 1; 423 | } 424 | } 425 | 426 | START_TEST(test_response_callback) 427 | { 428 | char *keys[64]; 429 | size_t key_lens[64]; 430 | size_t values_found = 0; 431 | omcache_t *oc = ot_init_omcache(1, LOG_INFO); 432 | ck_omcache_ok(omcache_set_response_callback(oc, test_response_callback_cb, &values_found)); 433 | for (int i = 0; i < 64; i ++) 434 | { 435 | key_lens[i] = asprintf(&keys[i], "test_response_callback_%d", i); 436 | // only set half of the keys 437 | if (i % 2) 438 | continue; 439 | ck_omcache(OMCACHE_BUFFERED, 440 | omcache_set(oc, (cuc *) keys[i], key_lens[i], (cuc *) keys[i], key_lens[i], 0, 0, 0, 0)); 441 | } 442 | ck_omcache_ok(omcache_set_buffering(oc, false)); 443 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 444 | 445 | // no even keys should be set 446 | omcache_req_t reqs[64]; 447 | size_t req_count = 64; 448 | ck_omcache_ok_or_again(omcache_get_multi(oc, (cuc **) keys, key_lens, 64, reqs, &req_count, NULL, NULL, 5000)); 449 | while (req_count > 0) 450 | ck_omcache_ok_or_again(omcache_io(oc, reqs, &req_count, NULL, NULL, 5000)); 451 | ck_assert_int_eq(values_found, 32); 452 | 453 | // now do the same thing with GAT (get-and-touch) 454 | req_count = 64; 455 | values_found = 0; 456 | time_t expirations[64]; 457 | for (int i = 0; i < 64; i ++) 458 | expirations[i] = 10 + i; 459 | ck_omcache_ok_or_again(omcache_gat_multi(oc, (cuc **) keys, key_lens, expirations, 64, reqs, &req_count, NULL, NULL, 5000)); 460 | while (req_count > 0) 461 | ck_omcache_ok_or_again(omcache_io(oc, reqs, &req_count, NULL, NULL, 5000)); 462 | ck_assert_int_eq(values_found, 32); 463 | 464 | for (int i = 0; i < 64; i ++) 465 | free(keys[i]); 466 | omcache_free(oc); 467 | } 468 | END_TEST 469 | 470 | Suite *ot_suite_commands(void) 471 | { 472 | Suite *s = suite_create("Commands"); 473 | 474 | ot_tcase_add(s, test_noop); 475 | ot_tcase_add(s, test_stats); 476 | ot_tcase_add(s, test_flush_all); 477 | ot_tcase_add(s, test_set_get_delete); 478 | ot_tcase_add(s, test_cas_and_flags); 479 | ot_tcase_add(s, test_add_and_replace); 480 | ot_tcase_add(s, test_append_and_prepend); 481 | ot_tcase_add_timeout(s, test_touch, 10); 482 | ot_tcase_add_timeout(s, test_gat, 10); 483 | ot_tcase_add(s, test_increment_and_decrement); 484 | ot_tcase_add(s, test_req_id_wraparound); 485 | ot_tcase_add(s, test_buffering); 486 | ot_tcase_add(s, test_response_callback); 487 | 488 | return s; 489 | } 490 | -------------------------------------------------------------------------------- /tests/test_failures.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include "test_omcache.h" 13 | #include 14 | #include 15 | #include 16 | 17 | #define TIMEOUT 2000 18 | 19 | START_TEST(test_suspended_memcache) 20 | { 21 | char strbuf[2048]; 22 | const unsigned char *val; 23 | size_t val_len, key_len; 24 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 25 | 26 | int susp_server_index = -1; 27 | int mc_port0, mc_port1, mc_port2; 28 | pid_t mc_pid0, mc_pid1, mc_pid2; 29 | 30 | mc_port0 = ot_start_memcached(NULL, &mc_pid0); 31 | mc_port1 = ot_start_memcached(NULL, &mc_pid1); 32 | mc_port2 = ot_start_memcached(NULL, &mc_pid2); 33 | 34 | sprintf(strbuf, "127.0.0.1:%d,127.0.0.1:%d,127.0.0.1:%d", mc_port0, mc_port1, mc_port2); 35 | ck_omcache_ok(omcache_set_servers(oc, strbuf)); 36 | ck_omcache_ok(omcache_set_dead_timeout(oc, 1000)); 37 | ck_omcache_ok(omcache_set_connect_timeout(oc, 3000)); 38 | ck_omcache_ok(omcache_set_reconnect_timeout(oc, 4000)); 39 | ck_omcache_ok(omcache_set_buffering(oc, true)); 40 | for (int i = 0; i < 1000; i ++) 41 | { 42 | key_len = snprintf(strbuf, sizeof(strbuf), "test_suspended_memcache_%d", i); 43 | ck_omcache(OMCACHE_BUFFERED, 44 | omcache_set(oc, (cuc *) strbuf, key_len, (cuc *) strbuf, key_len, 0, 0, 0, 0)); 45 | } 46 | ck_omcache_ok(omcache_set_buffering(oc, false)); 47 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 48 | 49 | // suspend one memcached and find out its server index 50 | kill(mc_pid2, SIGSTOP); 51 | usleep(100000); // allow 0.1 for SIGSTOP to be delivered 52 | for (int i = 0; i < 3; i ++) 53 | { 54 | omcache_server_info_t *sinfo = omcache_server_info(oc, i); 55 | if (sinfo->port == mc_port2) 56 | susp_server_index = i; 57 | ck_omcache_ok(omcache_server_info_free(oc, sinfo)); 58 | } 59 | ck_assert_int_ge(susp_server_index, 0); 60 | 61 | // get a key that belongs to the suspended server 62 | for (int i = 0; i < 1000; i ++) 63 | { 64 | key_len = snprintf(strbuf, sizeof(strbuf), "test_suspended_memcache_%d", i); 65 | if (omcache_server_index_for_key(oc, (cuc *) strbuf, key_len) == susp_server_index) 66 | break; 67 | *strbuf = 0; 68 | } 69 | ck_assert_int_ne(*strbuf, 0); 70 | 71 | // we'll get OMCACHE_SERVER_FAILURE after this server times out in 1 second 72 | int64_t begin = ot_msec(); 73 | ck_omcache(omcache_get(oc, (cuc *) strbuf, key_len, &val, &val_len, NULL, NULL, -1), OMCACHE_SERVER_FAILURE); 74 | ck_assert_int_le(ot_msec() - begin, 1500); 75 | begin = ot_msec(); 76 | ck_omcache(omcache_get(oc, (cuc *) strbuf, key_len, &val, &val_len, NULL, NULL, -1), OMCACHE_SERVER_FAILURE); 77 | ck_assert_int_le(ot_msec() - begin, 1500); 78 | // now the server should be disabled and fail faster with NOT_FOUND as we're not accessing the failed server anymore 79 | begin = ot_msec(); 80 | ck_omcache(omcache_get(oc, (cuc *) strbuf, key_len, &val, &val_len, NULL, NULL, -1), OMCACHE_NOT_FOUND); 81 | ck_assert_int_le(ot_msec() - begin, 500); 82 | ck_assert_int_ne(susp_server_index, omcache_server_index_for_key(oc, (cuc *) strbuf, key_len)); 83 | 84 | // wait for the server to come back online after reconnect timeout 85 | kill(mc_pid2, SIGCONT); 86 | sleep(5); 87 | // the first lookup should fail, but it should cause reconnection phase to start 88 | begin = ot_msec(); 89 | ck_omcache(omcache_get(oc, (cuc *) strbuf, key_len, &val, &val_len, NULL, NULL, -1), OMCACHE_NOT_FOUND); 90 | ck_assert_int_le(ot_msec() - begin, 500); 91 | // try to read the value a few times, this can fail a few more times 92 | // before the connection has been recreated and confirmed 93 | int ret = -1; 94 | for (int i = 0; (i < 10) && (ret != OMCACHE_OK); i ++) 95 | { 96 | if (i != 0 && ret != OMCACHE_OK) 97 | usleep(100000); 98 | ret = omcache_get(oc, (cuc *) strbuf, key_len, &val, &val_len, NULL, NULL, -1); 99 | } 100 | ck_assert_int_eq(ret, OMCACHE_OK); 101 | ck_assert_int_eq(susp_server_index, omcache_server_index_for_key(oc, (cuc *) strbuf, key_len)); 102 | 103 | omcache_free(oc); 104 | } 105 | END_TEST 106 | 107 | START_TEST(test_all_backends_fail) 108 | { 109 | size_t item_count = 10; 110 | const unsigned char keydata[] = 111 | "342f48a2c3a152a0fe39df4f2bca34d3c6c56e57797f0da682a6154ef7b674e3" 112 | "9c131c0c70442f94b865a5e0e030b48f4f51969fb80d5251fd67023c9982d3ab" 113 | "1ffd27717200ccb3c92882b10a04129422d5b71ddfaf24daf9fb5ee9cdfa2ef0"; 114 | size_t val_len; 115 | const unsigned char *val; 116 | pid_t mc_pid0, mc_pid1; 117 | int mc_port0 = ot_start_memcached(NULL, &mc_pid0); 118 | int mc_port1 = ot_start_memcached(NULL, &mc_pid1); 119 | char strbuf[100]; 120 | sprintf(strbuf, "127.0.0.1:%d,127.0.0.1:%d", mc_port0, mc_port1); 121 | 122 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 123 | ck_omcache_ok(omcache_set_servers(oc, strbuf)); 124 | ck_omcache_ok(omcache_set_dead_timeout(oc, 1000)); 125 | ck_omcache_ok(omcache_set_connect_timeout(oc, 2000)); 126 | ck_omcache_ok(omcache_set_reconnect_timeout(oc, 3000)); 127 | 128 | // send noops to both servers and write a bunch of values to them to make 129 | // sure we're connected to both servers and ketama picks both servers 130 | ck_omcache_ok(omcache_noop(oc, 0, 1000)); 131 | ck_omcache_ok(omcache_noop(oc, 1, 1000)); 132 | 133 | for (size_t i = 0; i < item_count; i ++) 134 | ck_omcache(omcache_set(oc, keydata + i, 100, keydata + i, 100, 0, 0, 0, 0), OMCACHE_BUFFERED); 135 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 136 | for (size_t i = 0; i < item_count; i ++) 137 | { 138 | ck_omcache_ok(omcache_get(oc, keydata + i, 100, &val, &val_len, NULL, NULL, 3000)); 139 | ck_assert_uint_eq(val_len, 100); 140 | ck_assert_int_eq(memcmp(val, keydata + i, 100), 0); 141 | } 142 | 143 | // suspend memcaches 144 | kill(mc_pid0, SIGSTOP); 145 | kill(mc_pid1, SIGSTOP); 146 | usleep(100000); // allow 0.1 for SIGSTOPs to be delivered 147 | 148 | // now try to read the values 149 | for (size_t i = 0; i < item_count; i ++) 150 | { 151 | int ret = omcache_get(oc, keydata + i, 100, &val, &val_len, NULL, NULL, 3000); 152 | ck_assert_int_ne(ret, OMCACHE_OK); 153 | } 154 | 155 | // sleep over timeouts again and try again 156 | sleep(3); 157 | 158 | // resume one memcache 159 | kill(mc_pid0, SIGCONT); 160 | 161 | // now try to read the values again, some of these will fail because we're 162 | // not yet fully connected to mc_pid0 and mc_pid1 is still down 163 | size_t found = 0; 164 | for (size_t i = 0; i < item_count; i ++) 165 | { 166 | int ret = omcache_get(oc, keydata + i, 100, &val, &val_len, NULL, NULL, 3000); 167 | if (ret == OMCACHE_OK) 168 | { 169 | ck_assert_uint_eq(val_len, 100); 170 | ck_assert_int_eq(memcmp(val, keydata + i, 100), 0); 171 | found ++; 172 | } 173 | else 174 | { 175 | usleep(1000); 176 | } 177 | } 178 | ck_assert_uint_ge(found, 1); // we should've found something 179 | 180 | // resume the other memcache 181 | kill(mc_pid1, SIGCONT); 182 | 183 | // sleep over timeouts again 184 | sleep(3); 185 | 186 | // try to read the values yet again, some of these will fail because we're 187 | // not yet fully connected after resuming mc_pid1 188 | found = 0; 189 | for (size_t i = 0; i < item_count; i ++) 190 | { 191 | int ret = omcache_get(oc, keydata + i, 100, &val, &val_len, NULL, NULL, 3000); 192 | if (ret == OMCACHE_OK) 193 | { 194 | ck_assert_uint_eq(val_len, 100); 195 | ck_assert_int_eq(memcmp(val, keydata + i, 100), 0); 196 | found ++; 197 | } 198 | else 199 | { 200 | usleep(1000); 201 | } 202 | } 203 | ck_assert_uint_ge(found, item_count / 2 + 1); // we should've found more than half 204 | 205 | // sleep over timeouts again 206 | sleep(3); 207 | 208 | // try to read the values a final time, now we should have everything 209 | for (size_t i = 0; i < item_count; i ++) 210 | { 211 | ck_omcache_ok(omcache_get(oc, keydata + i, 100, &val, &val_len, NULL, NULL, 3000)); 212 | ck_assert_uint_eq(val_len, 100); 213 | ck_assert_int_eq(memcmp(val, keydata + i, 100), 0); 214 | } 215 | 216 | omcache_free(oc); 217 | } 218 | END_TEST 219 | 220 | Suite *ot_suite_failures(void) 221 | { 222 | Suite *s = suite_create("Failures"); 223 | ot_tcase_add_timeout(s, test_suspended_memcache, 60); 224 | ot_tcase_add_timeout(s, test_all_backends_fail, 60); 225 | 226 | return s; 227 | } 228 | -------------------------------------------------------------------------------- /tests/test_libmcd_compat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * libmemcached compatibility test 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include "test_omcache.h" 13 | 14 | #ifdef WITH_LIBMEMCACHED 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #define ck_libmcd(c,e) cd_omcache((c), (e)) 21 | #define ck_libmcd_ok(c) ck_omcache((c), MEMCACHED_SUCCESS) 22 | 23 | static void set_n_values(omcache_t *oc, memcached_st *mc, 24 | char **keys_omc, char **keys_mcd, 25 | size_t *key_lens_omc, size_t *key_lens_mcd, 26 | char **keys_all, size_t *key_lens_all, 27 | size_t key_count) 28 | { 29 | ck_omcache_ok(omcache_set_buffering(oc, true)); 30 | for (size_t i = 0; i < key_count; i ++) 31 | { 32 | key_lens_omc[i] = asprintf(&keys_omc[i], "test_ketama_omc_%zu", i); 33 | ck_omcache(OMCACHE_BUFFERED, 34 | omcache_set(oc, (cuc *) keys_omc[i], key_lens_omc[i], (cuc *) keys_omc[i], key_lens_omc[i], 0, 0, 0, 0)); 35 | key_lens_mcd[i] = asprintf(&keys_mcd[i], "test_ketama_mcd_%zu", i); 36 | ck_libmcd_ok(memcached_set(mc, keys_mcd[i], key_lens_mcd[i], keys_mcd[i], key_lens_mcd[i], 0, 0)); 37 | keys_all[2 * i + 0] = keys_omc[i]; 38 | keys_all[2 * i + 1] = keys_mcd[i]; 39 | key_lens_all[2 * i + 0] = key_lens_omc[i]; 40 | key_lens_all[2 * i + 1] = key_lens_mcd[i]; 41 | } 42 | ck_omcache_ok(omcache_set_buffering(oc, false)); 43 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 44 | } 45 | 46 | static void check_n_values(omcache_t *oc, memcached_st *mc, 47 | char **keys, size_t *key_lens, 48 | size_t *omc_found, size_t *mcd_found, 49 | size_t key_count) 50 | { 51 | // check omcache 52 | omcache_value_t values[key_count]; 53 | size_t value_count = key_count; 54 | omcache_req_t reqs[key_count]; 55 | size_t req_count = key_count; 56 | ck_omcache_ok_or_again(omcache_get_multi(oc, (cuc **) keys, key_lens, key_count, reqs, &req_count, values, &value_count, 5000)); 57 | size_t omcache_values_found = value_count; 58 | while (req_count > 0) 59 | { 60 | int ret = OMCACHE_AGAIN; 61 | while (ret == OMCACHE_AGAIN) 62 | { 63 | value_count = key_count; 64 | ret = omcache_io(oc, reqs, &req_count, values, &value_count, 5000); 65 | ck_omcache_ok_or_again(ret); 66 | omcache_values_found += value_count; 67 | } 68 | } 69 | if (omc_found) 70 | *omc_found = omcache_values_found; 71 | else 72 | ck_assert_int_eq(omcache_values_found, key_count); 73 | 74 | // check libmemcached 75 | size_t libmemcached_values_found = 0; 76 | for (size_t i = 0; i < key_count; i ++) 77 | { 78 | memcached_return_t ret; 79 | char *value = memcached_get(mc, keys[i], key_lens[i], NULL, NULL, &ret); 80 | if (ret == MEMCACHED_SUCCESS) 81 | { 82 | libmemcached_values_found ++; 83 | free(value); 84 | } 85 | } 86 | if (mcd_found) 87 | *mcd_found = libmemcached_values_found; 88 | else 89 | ck_assert_int_eq(libmemcached_values_found, key_count); 90 | } 91 | 92 | static void test_libmemcached_ketama_compatibility_m(omcache_dist_t *omcache_method, 93 | memcached_behavior_t libmcd_method) 94 | { 95 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 96 | memcached_st *mc = memcached_create(NULL); 97 | char *keys_omc[1000], *keys_mcd[1000], *keys_all[2000]; 98 | size_t key_lens_omc[1000], key_lens_mcd[1000], key_lens_all[2000]; 99 | 100 | pid_t mc_pid0; // mc0 is our private memcached that we can break 101 | int mc_port0 = ot_start_memcached(NULL, &mc_pid0); 102 | int mc_port1 = ot_get_memcached(0); 103 | int mc_port2 = ot_get_memcached(1); 104 | 105 | char srvbuf[200]; 106 | sprintf(srvbuf, "127.0.0.1:%d,127.0.0.1:%d,127.0.0.1:%d", mc_port0, mc_port1, mc_port2); 107 | ck_omcache_ok(omcache_set_servers(oc, srvbuf)); 108 | ck_omcache_ok(omcache_set_dead_timeout(oc, 4000)); 109 | ck_omcache_ok(omcache_set_reconnect_timeout(oc, 3000)); 110 | ck_omcache_ok(omcache_set_distribution_method(oc, omcache_method)); 111 | 112 | ck_libmcd_ok(memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1)); 113 | ck_libmcd_ok(memcached_behavior_set(mc, libmcd_method, 1)); 114 | ck_libmcd_ok(memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_DEAD_TIMEOUT, 4)); 115 | ck_libmcd_ok(memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 3)); 116 | ck_libmcd_ok(memcached_server_add(mc, "127.0.0.1", mc_port0)); 117 | ck_libmcd_ok(memcached_server_add(mc, "127.0.0.1", mc_port1)); 118 | ck_libmcd_ok(memcached_server_add(mc, "127.0.0.1", mc_port2)); 119 | 120 | // set 1000 keys in memcached servers using omcache and libmemcached 121 | set_n_values(oc, mc, keys_omc, keys_mcd, key_lens_omc, key_lens_mcd, keys_all, key_lens_all, 1000); 122 | 123 | // verify that both clients can read all values set by both clients 124 | check_n_values(oc, mc, keys_omc, key_lens_omc, NULL, NULL, 1000); 125 | check_n_values(oc, mc, keys_mcd, key_lens_mcd, NULL, NULL, 1000); 126 | 127 | for (int i = 0; i < 2000; i ++) 128 | free(keys_all[i]); 129 | 130 | omcache_free(oc); 131 | memcached_free(mc); 132 | } 133 | 134 | START_TEST(test_ketama_compatibility) 135 | { 136 | // libmemcached doesn't provide a numeric version of the library, parse it 137 | // here to try to figure out which distribution algorithm it's using 138 | omcache_dist_t *dist = &omcache_dist_libmemcached_ketama; 139 | int v = strncmp(memcached_lib_version(), "1.0.", 4); 140 | if (v < 0 || (v == 0 && atoi(memcached_lib_version() + 4) < 10)) 141 | dist = &omcache_dist_libmemcached_ketama_pre1010; 142 | test_libmemcached_ketama_compatibility_m(dist, MEMCACHED_BEHAVIOR_KETAMA); 143 | } 144 | END_TEST 145 | 146 | START_TEST(test_ketama_weighted_compatibility) 147 | { 148 | test_libmemcached_ketama_compatibility_m( 149 | &omcache_dist_libmemcached_ketama_weighted, MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED); 150 | } 151 | END_TEST 152 | 153 | Suite *ot_suite_libmcd_compat(void) 154 | { 155 | Suite *s = suite_create("libmemcached compat"); 156 | ot_tcase_add_timeout(s, test_ketama_compatibility, 60); 157 | ot_tcase_add_timeout(s, test_ketama_weighted_compatibility, 60); 158 | return s; 159 | } 160 | #endif // WITH_LIBMEMCACHED 161 | -------------------------------------------------------------------------------- /tests/test_misc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include "test_omcache.h" 13 | #include "omcache_priv.h" 14 | 15 | START_TEST(test_strerror) 16 | { 17 | for (int i = OMCACHE_OK; i <= OMCACHE_SERVER_FAILURE; i ++) 18 | { 19 | const char *err = omcache_strerror(i); 20 | switch (i) 21 | { 22 | case OMCACHE_OK: 23 | case OMCACHE_NOT_FOUND: 24 | case OMCACHE_KEY_EXISTS: 25 | case OMCACHE_TOO_LARGE_VALUE: 26 | case OMCACHE_NOT_STORED: 27 | case OMCACHE_DELTA_BAD_VALUE: 28 | case OMCACHE_FAIL: 29 | case OMCACHE_AGAIN: 30 | case OMCACHE_INVALID: 31 | case OMCACHE_BUFFERED: 32 | case OMCACHE_BUFFER_FULL: 33 | case OMCACHE_NO_SERVERS: 34 | case OMCACHE_SERVER_FAILURE: 35 | ck_assert_int_ne(strcmp(err, "Unknown"), 0); 36 | break; 37 | default: 38 | ck_assert_int_eq(strcmp(err, "Unknown"), 0); 39 | } 40 | } 41 | } 42 | END_TEST 43 | 44 | START_TEST(test_md5) 45 | { 46 | const char text[] = "TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION"; 47 | const unsigned char text_md5[] = { 0xb9, 0x83, 0x21, 0xf1, 0x53, 0x89, 48 | 0xf7, 0xd0, 0x4a, 0x1e, 0x9a, 0x8d, 0x41, 0x40, 0x3b, 0x3b }; 49 | unsigned char buf[16]; 50 | omc_hash_md5((unsigned char *) text, strlen(text), buf); 51 | ck_assert_int_eq(memcmp(text_md5, buf, 16), 0); 52 | } 53 | END_TEST 54 | 55 | START_TEST(test_no_logging) 56 | { 57 | omcache_t *oc = ot_init_omcache(2, LOG_DEBUG); 58 | ck_omcache_ok(omcache_set_log_callback(oc, 999, NULL, NULL)); 59 | ck_omcache_ok(omcache_noop(oc, 0, -1)); 60 | ck_omcache_ok(omcache_noop(oc, 1, -1)); 61 | omcache_free(oc); 62 | } 63 | END_TEST 64 | 65 | 66 | Suite *ot_suite_misc(void) 67 | { 68 | Suite *s = suite_create("Misc"); 69 | ot_tcase_add(s, test_strerror); 70 | ot_tcase_add(s, test_md5); 71 | ot_tcase_add(s, test_no_logging); 72 | return s; 73 | } 74 | -------------------------------------------------------------------------------- /tests/test_omcache.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "test_omcache.h" 20 | #include "compat.h" 21 | 22 | struct mc_info_s { 23 | pid_t parent_pid; 24 | pid_t pid; 25 | unsigned short port; 26 | }; 27 | 28 | static struct mc_info_s memcacheds[1000]; 29 | static size_t memcached_count; 30 | static char memcached_version[100]; 31 | static char *memcached_path; 32 | 33 | const char *ot_memcached_version(void) 34 | { 35 | if (memcached_version[0]) 36 | return memcached_version; 37 | int iop[2]; 38 | pipe(iop); 39 | pid_t pid = fork(); 40 | if (pid == 0) 41 | { 42 | close(0); 43 | close(1); 44 | close(2); 45 | int fd = open("/dev/null", O_RDWR); 46 | dup2(fd, 0); 47 | dup2(fd, 2); 48 | close(fd); 49 | dup2(iop[1], fileno(stdout)); 50 | close(iop[1]); 51 | execl(memcached_path, "memcached", "-h", NULL); 52 | perror("execl"); 53 | _exit(1); 54 | } 55 | close(iop[1]); 56 | read(iop[0], memcached_version, sizeof(memcached_version)); 57 | close(iop[0]); 58 | memcached_version[sizeof(memcached_version) - 1] = 0; 59 | char *p = strchr(memcached_version, '\n'); 60 | if (p) 61 | *p = 0; 62 | if (memcmp(memcached_version, "memcached ", sizeof("memcached ") - 1) == 0) 63 | memmove(memcached_version, memcached_version + sizeof("memcached ") - 1, 20); 64 | return memcached_version; 65 | } 66 | 67 | int ot_get_memcached(size_t server_index) 68 | { 69 | while (server_index >= memcached_count) 70 | ot_start_memcached(NULL, NULL); 71 | return memcacheds[server_index].port; 72 | } 73 | 74 | int ot_start_memcached(const char *addr, pid_t *pidp) 75 | { 76 | if (memcached_count >= sizeof(memcacheds)/sizeof(memcacheds[0])) 77 | { 78 | printf("too many memcacheds running\n"); 79 | return -1; 80 | } 81 | struct timespec ts; 82 | clock_gettime(CLOCK_REALTIME, &ts); 83 | int port = 30000 + (ts.tv_nsec >> 10 & 0x7fff); 84 | pid_t pid = fork(); 85 | if (pid == 0) 86 | { 87 | char portbuf[32]; 88 | snprintf(portbuf, sizeof(portbuf), "%d", port); 89 | printf("Starting %s on port memcached %s\n", memcached_path, portbuf); 90 | execl(memcached_path, "memcached", "-vp", portbuf, "-l", addr ? addr : "127.0.0.1", NULL); 91 | perror("execl"); 92 | _exit(1); 93 | } 94 | // XXX: sleep 0.1s to allow memcached to start 95 | usleep(100000); 96 | memcacheds[memcached_count].parent_pid = getpid(); 97 | memcacheds[memcached_count].pid = pid; 98 | memcacheds[memcached_count++].port = port; 99 | if (pidp) 100 | *pidp = pid; 101 | return port; 102 | } 103 | 104 | static void kill_memcached(pid_t pid, int port) 105 | { 106 | printf("Sending SIGTERM to memcached pid %d on port %d\n", (int) pid, port); 107 | kill(pid, SIGTERM); 108 | } 109 | 110 | static void kill_memcacheds(void) 111 | { 112 | for (size_t i = 0; i < memcached_count; i ++) 113 | if (memcacheds[i].parent_pid == getpid()) 114 | kill_memcached(memcacheds[i].pid, memcacheds[i].port); 115 | } 116 | 117 | int ot_stop_memcached(int port) 118 | { 119 | for (size_t i = 0; i < memcached_count; i ++) 120 | { 121 | if (memcacheds[i].port != port) 122 | continue; 123 | kill_memcached(memcacheds[i].pid, memcacheds[i].port); 124 | memmove(&memcacheds[i], &memcacheds[--memcached_count], sizeof(memcacheds[i])); 125 | return 1; 126 | } 127 | return 0; 128 | } 129 | 130 | omcache_t *ot_init_omcache(int server_count, int log_level) 131 | { 132 | char srvstr[2048], *p = srvstr; 133 | omcache_t *oc = omcache_init(); 134 | omcache_set_log_callback(oc, log_level, omcache_log_stderr, NULL); 135 | if (server_count == 0) 136 | return oc; 137 | for (int i = 0; i < server_count; i ++) 138 | p += sprintf(p, "%s127.0.0.1:%d", i == 0 ? "" : ",", ot_get_memcached(i)); 139 | omcache_set_servers(oc, srvstr); 140 | return oc; 141 | } 142 | 143 | int64_t ot_msec(void) 144 | { 145 | struct timespec ts; 146 | clock_gettime(CLOCK_REALTIME, &ts); 147 | return ts.tv_sec * 1000 + ts.tv_nsec / 1000000; 148 | } 149 | 150 | int main(int argc, char **argv) 151 | { 152 | atexit(kill_memcacheds); 153 | 154 | // check logs to stdout, omcache to stderr 155 | setlinebuf(stdout); 156 | setlinebuf(stderr); 157 | 158 | memcached_path = getenv("MEMCACHED_PATH"); 159 | if (memcached_path == NULL) 160 | memcached_path = "/usr/bin/memcached"; 161 | 162 | // check memcached version (and presence) 163 | if (ot_memcached_version()[0] == 0) 164 | { 165 | fprintf(stderr, "%s: unable to determine memcached version from %s\n" 166 | "Set MEMCACHED_PATH environment variable to the right path\n", 167 | argv[0], memcached_path); 168 | return 1; 169 | } 170 | printf("memcached version: %s\n", ot_memcached_version()); 171 | 172 | // start two memcacheds in the parent process 173 | ot_start_memcached(NULL, NULL); 174 | ot_start_memcached(NULL, NULL); 175 | 176 | SRunner *sr = srunner_create(NULL); 177 | for (size_t i = 0; i < sizeof(suites) / sizeof(suites[0]); i ++) 178 | srunner_add_suite(sr, suites[i]()); 179 | 180 | srunner_set_log(sr, "-"); 181 | if (argc < 2) 182 | srunner_run_all(sr, CK_VERBOSE); 183 | else 184 | srunner_run(sr, NULL, argv[1], CK_VERBOSE); 185 | 186 | int number_failed = srunner_ntests_failed(sr); 187 | srunner_free(sr); 188 | 189 | return (number_failed == 0) ? 0 : 1; 190 | } 191 | -------------------------------------------------------------------------------- /tests/test_omcache.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #ifndef _TEST_OMCACHE_H 13 | #define _TEST_OMCACHE_H 1 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include "omcache.h" 21 | #include "compat.h" 22 | 23 | /* Check 0.9.8 which is available on Debian 6 and 7 doesn't support any 24 | * unsigned operations or greater / lesser comparisons, but as the macros 25 | * don't consider data types at all we can just use the int comparison for 26 | * everything there. 27 | * Also, older Check versions evaluate macro arguments multiple times. 28 | */ 29 | #if CHECK_MAJOR_VERSION * 10000 + CHECK_MINOR_VERSION * 100 + CHECK_MICRO_VERSION < 910 30 | #define ck_assert_int_ge(X, Y) _ck_assert_int(X, >=, Y) 31 | #define ck_assert_int_le(X, Y) _ck_assert_int(X, <=, Y) 32 | #define ck_assert_uint_eq(X, Y) _ck_assert_int(X, ==, Y) 33 | #define ck_assert_uint_ne(X, Y) _ck_assert_int(X, !=, Y) 34 | #define ck_assert_uint_ge(X, Y) _ck_assert_int(X, >=, Y) 35 | #define ck_assert_uint_le(X, Y) _ck_assert_int(X, <=, Y) 36 | #define ck_assert_ptr_eq(X, Y) _ck_assert_int(X, ==, Y) 37 | #define ck_assert_ptr_ne(X, Y) _ck_assert_int(X, !=, Y) 38 | #define ck_omcache(c,e) ({ int omc_ret = (c); ck_assert_int_eq(omc_ret, (e)); }) 39 | #define srunner_set_log(r,f) fprintf(stderr, "Check < 0.9.10 does not support logging to stdout.\n") 40 | #define srunner_run(r,s,c,v) ({ \ 41 | fprintf(stderr, "Check < 0.9.10 does not support selective running of tests.\n"); \ 42 | abort(); }) 43 | #else 44 | #define ck_omcache(c,e) ck_assert_int_eq((c), (e)) 45 | #endif 46 | 47 | #define ck_omcache_ok(c) ck_omcache((c), OMCACHE_OK) 48 | 49 | #define ck_omcache_ok_or_again(c) ({ int omc_ret1 = (c); if (omc_ret1 != OMCACHE_OK) ck_omcache(omc_ret1, OMCACHE_AGAIN); }) 50 | 51 | #define ot_tcase_add_timeout(s,f,t) \ 52 | ({ TCase *tc_ = tcase_create(#f); \ 53 | tcase_add_test(tc_, f); \ 54 | suite_add_tcase((s), tc_); \ 55 | if (t) tcase_set_timeout(tc_, (t)); \ 56 | }) 57 | #define ot_tcase_add(s,f) ot_tcase_add_timeout(s, f, 0) 58 | 59 | typedef const unsigned char cuc; 60 | 61 | omcache_t *ot_init_omcache(int server_count, int log_level); 62 | const char *ot_memcached_version(void); 63 | int ot_get_memcached(size_t server_index); 64 | int ot_start_memcached(const char *addr, pid_t *pid); 65 | int ot_stop_memcached(int port); 66 | int64_t ot_msec(void); 67 | 68 | Suite *ot_suite_commands(void); 69 | Suite *ot_suite_failures(void); 70 | #ifdef WITH_LIBMEMCACHED 71 | Suite *ot_suite_libmcd_compat(void); 72 | #endif // WITH_LIBMEMCACHED 73 | Suite *ot_suite_misc(void); 74 | Suite *ot_suite_servers(void); 75 | 76 | typedef Suite *(suite_init_cb)(void); 77 | static suite_init_cb *const suites[] = { 78 | ot_suite_commands, 79 | ot_suite_failures, 80 | #ifdef WITH_LIBMEMCACHED 81 | ot_suite_libmcd_compat, 82 | #endif // WITH_LIBMEMCACHED 83 | ot_suite_misc, 84 | ot_suite_servers, 85 | }; 86 | 87 | #endif // !_TEST_OMCACHE_H 88 | -------------------------------------------------------------------------------- /tests/test_servers.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Unit tests for OMcache 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include "test_omcache.h" 13 | #include 14 | 15 | 16 | START_TEST(test_server_list) 17 | { 18 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 19 | ck_assert_int_eq(omcache_server_index_for_key(oc, (cuc *) "foo", 3), 0); 20 | ck_assert_ptr_eq(omcache_server_info(oc, 0), NULL); 21 | // NOTE: omcache sorts server list internally, host and portnames are not 22 | // checked when server list is created, they're resolved when we actually 23 | // try to connect so invalid entries can be pushed to the list 24 | ck_omcache_ok(omcache_set_servers(oc, 25 | "foo:bar, [::1]:11211, [fe80::5054:ff:fefb:beef], 8.8.8.8:22,, " 26 | "127.0.0.1:11300 , 10.0.0.0, 10.10.10.10:11111")); 27 | ck_omcache_ok(omcache_set_servers(oc, 28 | "127.0.0.1:11300, 10.0.0.0, [::1]:11111, 192.168.255.255:99999")); 29 | for (int i = 0; i < 4; i ++) 30 | { 31 | omcache_server_info_t *sinfo = omcache_server_info(oc, i); 32 | ck_assert_ptr_ne(sinfo, NULL); 33 | ck_omcache(sinfo->omcache_version, OMCACHE_VERSION); 34 | ck_assert_int_eq(sinfo->server_index, i); 35 | switch (i) 36 | { 37 | case 0: 38 | ck_assert_int_eq(sinfo->port, 11211); 39 | ck_assert_str_eq(sinfo->hostname, "10.0.0.0"); 40 | break; 41 | case 1: 42 | ck_assert_int_eq(sinfo->port, 11300); 43 | ck_assert_str_eq(sinfo->hostname, "127.0.0.1"); 44 | break; 45 | case 2: 46 | ck_assert_int_eq(sinfo->port, 99999); 47 | ck_assert_str_eq(sinfo->hostname, "192.168.255.255"); 48 | break; 49 | case 3: 50 | ck_assert_int_eq(sinfo->port, 11111); 51 | ck_assert_str_eq(sinfo->hostname, "::1"); 52 | break; 53 | default: 54 | ck_assert(false); 55 | } 56 | ck_omcache_ok(omcache_server_info_free(oc, sinfo)); 57 | } 58 | ck_omcache_ok(omcache_set_servers(oc, "")); 59 | omcache_free(oc); 60 | } 61 | END_TEST 62 | 63 | static void check_distribution(omcache_t *oc, int srvcnt) 64 | { 65 | int hits[srvcnt]; 66 | memset(&hits, 0, sizeof(hits)); 67 | for (int i = 0; i < 1000; i ++) 68 | { 69 | int si = omcache_server_index_for_key(oc, (cuc *) &i, sizeof(i)); 70 | ck_assert_int_ge(si, 0); 71 | ck_assert_int_le(si, 3); 72 | hits[si] ++; 73 | } 74 | for (int i = 0; i < srvcnt; i++) 75 | { 76 | ck_assert_int_ge(hits[i], 200); 77 | ck_assert_int_le(hits[i], 300); 78 | } 79 | } 80 | 81 | START_TEST(test_distribution) 82 | { 83 | // check that we have a mostly even distribution with all algorithms 84 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 85 | ck_omcache_ok(omcache_set_servers(oc, "127.0.0.1:1, 127.0.0.1:2, 127.0.0.1:3, 127.0.0.1:4")); 86 | ck_omcache_ok(omcache_set_distribution_method(oc, &omcache_dist_libmemcached_ketama)); 87 | check_distribution(oc, 4); 88 | ck_omcache_ok(omcache_set_distribution_method(oc, &omcache_dist_libmemcached_ketama_weighted)); 89 | check_distribution(oc, 4); 90 | ck_omcache_ok(omcache_set_distribution_method(oc, &omcache_dist_libmemcached_ketama_pre1010)); 91 | check_distribution(oc, 4); 92 | omcache_free(oc); 93 | } 94 | END_TEST 95 | 96 | START_TEST(test_no_servers) 97 | { 98 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 99 | ck_omcache(omcache_noop(oc, 0, 0), OMCACHE_NO_SERVERS); 100 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 0)); 101 | omcache_free(oc); 102 | } 103 | END_TEST 104 | 105 | START_TEST(test_invalid_servers) 106 | { 107 | omcache_t *oc = ot_init_omcache(0, LOG_INFO); 108 | ck_omcache_ok(omcache_set_servers(oc, "127.0.0.1:1, 127.0.0.1:22, 127.0.0.foobar:asdf,,,")); 109 | 110 | // since omcache starts with good faith in servers it's given it won't 111 | // actually notice that the first two servers are dead or not talking 112 | // memcached protocol before trying to communicate with them and then 113 | // dropping the connections. the third one will fail immediately as it's 114 | // address can't be resolved. 115 | ck_omcache(omcache_noop(oc, 0, 2000), OMCACHE_NO_SERVERS); 116 | ck_omcache(omcache_noop(oc, 1, 2000), OMCACHE_NO_SERVERS); 117 | ck_omcache(omcache_noop(oc, 1, 2000), OMCACHE_NO_SERVERS); 118 | // when built with asyncns it can take one more round of calls for servers to be declared dead 119 | int ret = omcache_get(oc, (cuc *) "foo", 3, NULL, NULL, NULL, NULL, 2000); 120 | if (ret != OMCACHE_SERVER_FAILURE) 121 | ck_assert_int_eq(ret, OMCACHE_NO_SERVERS); 122 | 123 | omcache_free(oc); 124 | } 125 | END_TEST 126 | 127 | START_TEST(test_multiple_times_same_server) 128 | { 129 | omcache_t *oc = ot_init_omcache(1, LOG_INFO); 130 | char sbuf[sizeof("127.0.0.1:11211,") * 20 + 1], *p = sbuf; 131 | 132 | ck_omcache(omcache_get(oc, (cuc *) "foo", 3, NULL, NULL, NULL, NULL, 2000), OMCACHE_NOT_FOUND); 133 | for (int i = 0; i < 20; i++) 134 | p += sprintf(p, "%s127.0.0.1:%d", i == 0 ? "" : ",", ot_get_memcached(0)); 135 | ck_omcache_ok(omcache_set_servers(oc, sbuf)); 136 | ck_omcache(omcache_get(oc, (cuc *) "foo", 3, NULL, NULL, NULL, NULL, 2000), OMCACHE_NOT_FOUND); 137 | for (int i = 0; i < 20; i++) 138 | ck_omcache_ok(omcache_noop(oc, i, 1000)); 139 | 140 | omcache_free(oc); 141 | } 142 | END_TEST 143 | 144 | START_TEST(test_fd_map_allocations) 145 | { 146 | omcache_t *oc = ot_init_omcache(1, LOG_INFO); 147 | char sbuf[sizeof("127.0.0.1:11211,") * 35 + 1], *p = sbuf; 148 | int first_pipes[2], pipes[2]; 149 | 150 | // create some pipes to use up filedescriptors 151 | pipe(first_pipes); 152 | 153 | for (int i = 0; i < 20; i++) 154 | { 155 | pipe(pipes); // allocate some more pipes 156 | p += sprintf(p, "%s127.0.0.1:%d", i == 0 ? "" : ",", ot_get_memcached(0)); 157 | } 158 | ck_omcache_ok(omcache_set_servers(oc, sbuf)); 159 | ck_omcache_ok(omcache_noop(oc, 4, 1000)); 160 | 161 | ck_omcache_ok(omcache_set_buffering(oc, true)); 162 | for (int i = 0; i < 20; i++) 163 | ck_omcache_ok(omcache_noop(oc, i, 100)); 164 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 165 | 166 | // close the early pipes 167 | close(first_pipes[0]); 168 | close(first_pipes[1]); 169 | 170 | // add another server 171 | p += sprintf(p, ",127.0.0.1:%d", ot_get_memcached(1)); 172 | ck_omcache_ok(omcache_set_servers(oc, sbuf)); 173 | for (int i = 0; i < 21; i++) 174 | ck_omcache_ok(omcache_noop(oc, i, 100)); 175 | ck_omcache_ok(omcache_io(oc, NULL, NULL, NULL, NULL, 5000)); 176 | 177 | omcache_free(oc); 178 | } 179 | END_TEST 180 | 181 | START_TEST(test_ipv6) 182 | { 183 | // NOTE: memcached doesn't support specifying a literal IPv6 address on 184 | // the commandline, so we try to use 'localhost6' if we can find it in 185 | // /etc/hosts 186 | char linebuf[300]; 187 | bool have_localhost6 = false; 188 | FILE *fp = fopen("/etc/hosts", "r"); 189 | if (fp == NULL) 190 | return; 191 | while (!have_localhost6) 192 | { 193 | if (fgets(linebuf, sizeof(linebuf), fp) == NULL) 194 | break; 195 | if (strstr(linebuf, "localhost6") != NULL) 196 | have_localhost6 = true; 197 | } 198 | fclose(fp); 199 | if (!have_localhost6) 200 | return; 201 | omcache_t *oc = ot_init_omcache(0, LOG_DEBUG); 202 | int mc_port = ot_start_memcached("localhost6", NULL); 203 | sprintf(linebuf, "localhost6:%d", mc_port); 204 | ck_omcache_ok(omcache_set_servers(oc, linebuf)); 205 | ck_omcache_ok(omcache_noop(oc, 0, 1000)); 206 | omcache_free(oc); 207 | } 208 | END_TEST 209 | 210 | Suite *ot_suite_servers(void) 211 | { 212 | Suite *s = suite_create("Servers"); 213 | 214 | ot_tcase_add(s, test_server_list); 215 | ot_tcase_add(s, test_distribution); 216 | ot_tcase_add(s, test_no_servers); 217 | ot_tcase_add(s, test_invalid_servers); 218 | ot_tcase_add(s, test_multiple_times_same_server); 219 | ot_tcase_add(s, test_fd_map_allocations); 220 | ot_tcase_add(s, test_ipv6); 221 | 222 | return s; 223 | } 224 | -------------------------------------------------------------------------------- /util.c: -------------------------------------------------------------------------------- 1 | /* 2 | * OMcache: internal utility functions 3 | * 4 | * Copyright (c) 2014, Oskari Saarenmaa 5 | * All rights reserved. 6 | * 7 | * This file is under the Apache License, Version 2.0. 8 | * See the file `LICENSE` for details. 9 | * 10 | */ 11 | 12 | #include 13 | #include 14 | #include "omcache_priv.h" 15 | 16 | // murmurhash3's finalization function 17 | static inline uint32_t 18 | omc_hash_uint32(uint32_t val) 19 | { 20 | val ^= val >> 16; 21 | val *= 0x85ebca6b; 22 | val ^= val >> 13; 23 | val *= 0xc2b2ae35; 24 | val ^= val >> 16; 25 | return val; 26 | } 27 | 28 | static omc_hash_table_t *omc_hash_table_reset(omc_hash_table_t *hash) 29 | { 30 | hash->count = 0; 31 | hash->freelist = NULL; 32 | for (uint32_t i = 0; i < hash->size; i ++) 33 | { 34 | hash->buckets[i] = NULL; 35 | hash->nodes[i].key = -1; 36 | hash->nodes[i].val = hash->not_found_val; 37 | hash->nodes[i].next = hash->freelist; 38 | hash->freelist = &hash->nodes[i]; 39 | } 40 | return hash; 41 | } 42 | 43 | omc_hash_table_t *omc_hash_table_init(omc_hash_table_t *old_hash, uint32_t size, void *not_found_val) 44 | { 45 | // reuse old hash table if one was given and the size is right 46 | if (old_hash != NULL) 47 | { 48 | if (old_hash->size >= size) 49 | return omc_hash_table_reset(old_hash); 50 | omc_hash_table_free(old_hash); 51 | } 52 | // allocate the hash table and enough memory to hold all the nodes and 53 | // pointers to bucket heads in a single allocation 54 | size_t struct_alloc = sizeof(omc_hash_table_t); 55 | size_t nodes_alloc = size * sizeof(omc_hash_node_t); 56 | size_t pointers_alloc = size * sizeof(omc_hash_node_t *); 57 | omc_hash_table_t *hash = malloc(struct_alloc + nodes_alloc + pointers_alloc); 58 | hash->not_found_val = not_found_val; 59 | hash->size = size; 60 | hash->buckets = (omc_hash_node_t **) ( 61 | ((unsigned char *) hash) + struct_alloc + nodes_alloc); 62 | return omc_hash_table_reset(hash); 63 | } 64 | 65 | void omc_hash_table_free(omc_hash_table_t *hash) 66 | { 67 | free(hash); 68 | } 69 | 70 | void *omc_hash_table_find(omc_hash_table_t *hash, uint32_t key) 71 | { 72 | uint32_t bucket = omc_hash_uint32(key) % hash->size; 73 | for (omc_hash_node_t *n = hash->buckets[bucket]; n != NULL; n = n->next) 74 | if (n->key == key) 75 | return n->val; 76 | return hash->not_found_val; 77 | } 78 | 79 | int omc_hash_table_add(omc_hash_table_t *hash, uint32_t key, void *val) 80 | { 81 | if (hash->count == hash->size) 82 | return -1; 83 | uint32_t bucket = omc_hash_uint32(key) % hash->size; 84 | omc_hash_node_t *n = hash->freelist; 85 | hash->freelist = n->next; 86 | n->next = hash->buckets[bucket]; 87 | n->key = key; 88 | n->val = val; 89 | hash->buckets[bucket] = n; 90 | hash->count ++; 91 | return 0; 92 | } 93 | 94 | void *omc_hash_table_del(omc_hash_table_t *hash, uint32_t key) 95 | { 96 | uint32_t bucket = omc_hash_uint32(key) % hash->size; 97 | omc_hash_node_t *p = NULL; 98 | for (omc_hash_node_t *n = hash->buckets[bucket]; n != NULL; n = n->next) 99 | { 100 | if (n->key == key) 101 | { 102 | void *val = n->val; 103 | if (p) 104 | p->next = n->next; 105 | else 106 | hash->buckets[bucket] = n->next; 107 | n->key = -1; 108 | n->val = hash->not_found_val; 109 | n->next = hash->freelist; 110 | hash->freelist = n; 111 | hash->count --; 112 | return val; 113 | } 114 | p = n; 115 | } 116 | return hash->not_found_val; 117 | } 118 | -------------------------------------------------------------------------------- /version.mk: -------------------------------------------------------------------------------- 1 | short_ver = 0.3.0 2 | long_ver = $(shell git describe --long 2>/dev/null || echo $(short_ver)-0-unknown-g`git describe --always`) 3 | --------------------------------------------------------------------------------