├── .gitignore ├── LICENSE ├── Makefile ├── README.org ├── c_src ├── build_deps.sh ├── embedded_innodb-1.0.6.6750.tar.gz ├── innostore_drv.c ├── innostore_drv.h └── patches │ └── disable_verbose_warnings.patch ├── ebin └── innostore.app ├── package └── Makefile ├── priv ├── innodump ├── innoload └── riak-innostore ├── rebar ├── rebar.config └── src ├── innostore.erl ├── innostore_riak.erl └── riak_kv_innostore_backend.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit/ 2 | ebin/ 3 | embedded_innodb-*/ 4 | c_src/innodb/lib/ 5 | c_src/innostore_drv.o 6 | priv/innostore_drv.so -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | ifeq ($(RIAK),) 3 | RIAK = "$(OTPROOT)" 4 | endif 5 | 6 | REPO ?= innostore 7 | INNOSTORE_TAG = $(shell git describe --tags) 8 | RELEASE_NAME = $(REPO)-$(INNOSTORE_TAG) 9 | INSTALL_ERR = "$(RIAK) doesn't exist or isn't writable." 10 | TARGET_ERR = "Error: no target location defined - set RIAK or OTPROOT" 11 | INSTALL_MSG = "Installing to $(RIAK)/lib..." 12 | 13 | .PHONY: package pkgclean distclean 14 | 15 | all: compile 16 | 17 | compile: 18 | ./rebar compile 19 | 20 | clean: 21 | ./rebar clean 22 | 23 | deps: 24 | ./rebar get-deps 25 | 26 | install: 27 | @[ "$(RIAK)" == "" ] && echo "$(TARGET_ERR)" && exit 2 || echo "$(INSTALL_MSG)" 28 | @[ -w "$(RIAK)" ] && cp -r `pwd` "$(RIAK)/lib" || echo "$(INSTALL_ERR)" 29 | @: 30 | 31 | test: 32 | ./rebar eunit 33 | 34 | # Release tarball creation 35 | # Generates a tarball that includes all the deps sources so no checkouts are necessary 36 | archivegit = git archive --format=tar --prefix=$(1)/ HEAD | (cd $(2) && tar xf -) 37 | archive = $(call archivegit,$(1),$(2)) 38 | 39 | buildtar = mkdir distdir && \ 40 | git clone . distdir/$(REPO)-clone && \ 41 | cd distdir/$(REPO)-clone && \ 42 | git checkout $(INNOSTORE_TAG) && \ 43 | $(call archive,$(RELEASE_NAME),..) && cd .. 44 | 45 | distdir: 46 | $(if $(INNOSTORE_TAG), $(call buildtar), $(error "You can't generate a release tarball from a non-tagged revision. Run 'git checkout ', then 'make dist'")) 47 | 48 | dist $(RELEASE_NAME).tar.gz: distdir 49 | cd distdir; \ 50 | tar czf ../$(RELEASE_NAME).tar.gz $(RELEASE_NAME) 51 | 52 | distclean: ballclean 53 | 54 | ballclean: 55 | rm -rf $(RELEASE_NAME).tar.gz distdir 56 | 57 | package: dist 58 | $(MAKE) -C package package 59 | 60 | pkgclean: 61 | $(MAKE) -C package pkgclean 62 | 63 | export INNOSTORE_TAG REPO RELEASE_NAME 64 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | * innostore 2 | ** Overview 3 | Innostore is a simple Erlang API to Embedded InnoDB. 4 | 5 | Innostore does not attempt to provide the same API as Inno itself, but 6 | instead provides a simpler interface which is sufficient for many 7 | uses. This interface is exported by the =innostore= module of the 8 | =innostore= application. 9 | 10 | An example of use of that interface can be found in the 11 | =innostore_riak= module, which is a valid Riak backend module using 12 | innostore for storage. 13 | 14 | ** Installation 15 | 1. Download or clone the innostore project from 16 | http://github.com/basho/innostore. 17 | 18 | 2. In the =innostore= directory, build it: 19 | : $ make 20 | 21 | 3. Install innostore (may need root privs)... 22 | 1. Option 1: ... into your Erlang distribution: 23 | : $ make install 24 | 25 | This will install innostore to $OTPROOT/lib, so be sure you have 26 | it set in your environment. 27 | 28 | 2. Option 2: ... into an existing Riak build: 29 | : $ RIAK=/usr/lib/riak make install 30 | 31 | This will allow you to custom install innostore into the lib 32 | folder that Riak is using. 33 | 34 | 4. Finally, configure riak to use innostore in =app.config= by setting 35 | the =storage_backend=. Change =riak/etc/app.config=: 36 | 37 | : {storage_backend, riak_kv_innostore_backend} 38 | 39 | 5. You may wish to also tune the innostore engine. Add an =innostore= 40 | application section to the =riak/etc/app.config=: 41 | 42 | #+BEGIN_SRC erlang 43 | %% Inno db config 44 | {innostore, [ 45 | {data_home_dir, "/mnt/innodb"}, 46 | {log_group_home_dir, "/mnt/innodb"}, 47 | {buffer_pool_size, 2147483648} %% 2G of buffer 48 | ]} 49 | #+END_SRC 50 | 51 | ** Tuning 52 | 53 | While InnoDB can be extremely fast for a durable store, its 54 | performance is highly dependent on tuning the configuration to match 55 | the hardware and dataset. All the configuration is available as 56 | application variables in the =innostore= application scope. An example 57 | configuration follows: 58 | 59 | #+BEGIN_SRC erlang 60 | %% InnoDB config 61 | {innostore, [ 62 | %% Where data files go 63 | {data_home_dir, "/innodb"}, 64 | 65 | %% Where log files go 66 | {log_group_home_dir, "/innodb-log"}, 67 | 68 | %% 2G in-memory buffer in bytes 69 | {buffer_pool_size, 2147483648}, 70 | 71 | %% How many files you need -- usually, 3 < x < 6 72 | {log_files_in_group, 6}, 73 | 74 | %% No bigger than 256MB 75 | {log_file_size, 134217728} 76 | ]}, 77 | #+END_SRC 78 | 79 | In general, only the first three parameters (=data_home_dir=, 80 | =log_group_home_dir= and =buffer_pool_size=) will need to be 81 | changed. It is strongly recommended that the data home dir and log 82 | home dir are kept on separate spindles/drives. 83 | 84 | The =buffer_pool_size= determines how much data Inno tries to keep in 85 | memory at once. Obviously, the more of your dataset that can fit in 86 | RAM, the better Inno will perform. If you are running a 64-bit Erlang 87 | VM, the =buffer_pool_size= can safely be set above 2G. 88 | 89 | You can control the number of file descriptors Inno will use with 90 | =open_files=. Inno defaults to 300 which can cause problems on some 91 | platforms (e.g. OS X has a default limit of 256 handles). Either set 92 | ={open_files, 100}= in the config or increase the number of handles 93 | available. 94 | 95 | Make sure to set =noatime= on any disk involved with Inno, 96 | particularly if you are expecting significant load. 97 | 98 | When running innostore on solaris+zfs, make sure to set the 99 | =recordsize=16k= on the pool where =data_home_dir= lives (prior to 100 | starting innostore). You may also find that setting 101 | =primarycache=metadata= will positively influence performance. 102 | 103 | ** Logging 104 | 105 | By default log output from Inno will be delivered on stderr. The 106 | =error_log= config option allows redirecting to a file, for example to 107 | redirect to =/var/log/innostore.log= 108 | 109 | : {innostore, [{error_log, "/var/log/innostore.log"}]}. 110 | 111 | ** InnoDB Table Format 112 | 113 | Embedded InnoDb offers several table formats: =compact=, =dynamic= and 114 | =compressed=. 115 | 116 | - =compact= format stores the first 768 bytes of the value with the 117 | key and any extra data on extension pages. This is a good option 118 | for small values. 119 | - =dynamic= format stores the value outside separately from the 120 | index. For larger values (>600 bytes) this will make the index 121 | pages denser and may provide a performance increase. 122 | - =compressed= format is like dynamic format, except it compresses 123 | the key and data pages. 124 | 125 | The default format is =compact= to match previous innostore 126 | releases. To configure, set the =format= option in your innostore 127 | config. The setting is system wide and will be used for all buckets. 128 | 129 | #+BEGIN_SRC erlang 130 | %% Use dynamic format tables. 131 | {innostore, [{format, dynamic}]}. 132 | #+END_SRC 133 | 134 | If you wish to use compressed tables the =page_size= must be set to 0 135 | (changing this is not recommended for any other case). 136 | 137 | #+BEGIN_SRC erlang 138 | {innostore, [{format, compressed}, 139 | {page_size, 0}]}. 140 | #+END_SRC 141 | 142 | ** Contributing 143 | We encourage contributions to =innostore= from the community. 144 | 145 | 1) Fork the =innostore= repository on 146 | [[https://github.com/basho/innostore][Github]]. 147 | 2) Clone your fork or add the remote if you already have a clone of 148 | the repository. 149 | #+BEGIN_SRC shell 150 | git clone git@github.com:yourusername/innostore.git 151 | # or 152 | git remote add mine git@github.com:yourusername/innostore.git 153 | #+END_SRC 154 | 3) Create a topic branch for your change. 155 | #+BEGIN_SRC shell 156 | git checkout -b some-topic-branch 157 | #+END_SRC 158 | 4) Make your change and commit. Use a clear and descriptive commit 159 | message, spanning multiple lines if detailed explanation is 160 | needed. 161 | 5) Push to your fork of the repository and then send a pull-request 162 | through Github. 163 | #+BEGIN_SRC shell 164 | git push mine some-topic-branch 165 | #+END_SRC 166 | 6) A Basho engineer or community maintainer will review your patch 167 | and merge it into the main repository or send you feedback. 168 | -------------------------------------------------------------------------------- /c_src/build_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | INNO_VSN=1.0.6.6750 6 | 7 | if [ `basename $PWD` != "c_src" ]; then 8 | pushd c_src 9 | fi 10 | 11 | BASEDIR="$PWD" 12 | 13 | case "$1" in 14 | clean) 15 | rm -rf innodb embedded_innodb-$INNO_VSN 16 | ;; 17 | 18 | *) 19 | test -f innodb/lib/libinnodb.a && exit 0 20 | 21 | tar -xzf embedded_innodb-$INNO_VSN.tar.gz 22 | for x in patches/*; do 23 | echo $x 24 | patch -p0 < $x 25 | done 26 | 27 | (cd embedded_innodb-$INNO_VSN && \ 28 | ./configure --disable-shared --enable-static --with-pic \ 29 | --prefix=$BASEDIR/innodb && \ 30 | make && make install) 31 | 32 | ;; 33 | esac 34 | 35 | -------------------------------------------------------------------------------- /c_src/embedded_innodb-1.0.6.6750.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/innostore/fd487cd0c2ce220b25f007450b1a29e81bb05d17/c_src/embedded_innodb-1.0.6.6750.tar.gz -------------------------------------------------------------------------------- /c_src/innostore_drv.c: -------------------------------------------------------------------------------- 1 | // ------------------------------------------------------------------- 2 | // 3 | // innostore: Simple Erlang API to Embedded Inno DB 4 | // 5 | // Copyright (c) 2010 Basho Technologies, Inc. All Rights Reserved. 6 | // 7 | // innostore is free software: you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by 9 | // the Free Software Foundation, either version 2 of the License, or 10 | // (at your option) any later version. 11 | // 12 | // innostore is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with innostore. If not, see . 19 | // 20 | // ------------------------------------------------------------------- 21 | 22 | #include "innostore_drv.h" 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | /** 32 | * Erlang driver functions 33 | */ 34 | static int innostore_drv_init(); 35 | 36 | static ErlDrvData innostore_drv_start(ErlDrvPort port, char* buffer); 37 | 38 | static void innostore_drv_stop(ErlDrvData handle); 39 | 40 | static void innostore_drv_finish(); 41 | 42 | static int innostore_drv_control(ErlDrvData handle, unsigned int cmd, 43 | char* inbuf, int inbuf_sz, 44 | char** outbuf, int outbuf_sz); 45 | 46 | /** 47 | * Erlang Driver Entry 48 | */ 49 | ErlDrvEntry innostore_drv_entry = 50 | { 51 | innostore_drv_init, /* F_PTR init, called when library loaded */ 52 | innostore_drv_start, /* L_PTR start, called when port is opened */ 53 | innostore_drv_stop, /* F_PTR stop, called when port is closed */ 54 | NULL, /* F_PTR output, called when erlang has sent */ 55 | NULL, /* F_PTR ready_input, called when input descriptor ready */ 56 | NULL, /* F_PTR ready_output, called when output descriptor ready */ 57 | "innostore_drv", /* driver_name */ 58 | innostore_drv_finish, /* F_PTR finish, called when unloaded */ 59 | NULL, /* handle */ 60 | innostore_drv_control, /* F_PTR control, port_command callback */ 61 | NULL, /* F_PTR timeout, reserved */ 62 | NULL, /* F_PTR outputv, reserved */ 63 | NULL, /* F_PTR ready_async */ 64 | NULL, /* F_PTR flush */ 65 | NULL, /* F_PTR call */ 66 | NULL, /* F_PTR event */ 67 | ERL_DRV_EXTENDED_MARKER, 68 | ERL_DRV_EXTENDED_MAJOR_VERSION, 69 | ERL_DRV_EXTENDED_MINOR_VERSION, 70 | ERL_DRV_FLAG_USE_PORT_LOCKING, 71 | NULL, /* Reserved */ 72 | NULL /* F_PTR process_exit */ 73 | }; 74 | 75 | 76 | /** 77 | * Core function prototypes 78 | */ 79 | static void* innostore_worker(void* arg); 80 | 81 | static void do_set_cfg(void* arg); 82 | static void do_start(void* arg); 83 | static void do_init_table(void* arg); 84 | static void do_get(void* arg); 85 | static void do_put(void* arg); 86 | static void do_delete(void* arg); 87 | 88 | static void do_list_tables(void* arg); 89 | static int do_list_tables_cb(void* arg, const char* tablename, int tablename_sz); 90 | 91 | static void do_cursor_open(void* arg); 92 | static void do_cursor_move(void* arg); 93 | static void do_cursor_read(unsigned int content_flag, PortState* state); 94 | static void do_cursor_close(void* arg); 95 | 96 | static void do_drop_table(void* arg); 97 | static void do_status(void* arg); 98 | 99 | static void send_ok(PortState* state); 100 | static void send_ok_atom(PortState* state, const char* atom); 101 | static void send_error_atom(PortState* state, const char* atom); 102 | static void send_error_str(PortState* state, const char* str); 103 | 104 | /** 105 | * Compression layer functions and defines 106 | */ 107 | #define COMPRESSION_NONE 0 108 | 109 | static int compress(unsigned int cflag, 110 | PortState* state, char* in_value, unsigned int in_value_sz, 111 | char** out_value, unsigned int* out_value_sz); 112 | 113 | static int decompress(PortState* state, char* in_value, unsigned int in_value_sz, 114 | char** out_value, unsigned int* out_value_sz); 115 | 116 | /** 117 | * Logging 118 | */ 119 | #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(__GNUC__) 120 | # define log(...) G_LOGGER_FN(G_LOGGER_FH, __VA_ARGS__) 121 | #else 122 | # define log(X) G_LOGGER_FN(G_LOGGER_FH, X) 123 | #endif 124 | 125 | static int set_log_file(const char* filename); 126 | static int raw_logger(ib_msg_stream_t stream, const char* fmt, ...); 127 | 128 | 129 | /** 130 | * Globals for inno mgmt -- need to ensure engine only gets started/configured once per VM. 131 | */ 132 | #define ENGINE_STOPPED 0 133 | #define ENGINE_STARTING 1 134 | #define ENGINE_STARTED 2 135 | 136 | static int G_ENGINE_STATE = 0; 137 | static ErlDrvMutex* G_ENGINE_STATE_LOCK; 138 | 139 | /** 140 | * Globals for inno logging */ 141 | 142 | static ErlDrvMutex* G_LOGGER_LOCK; 143 | static char* G_LOGGER_BUF = NULL; 144 | static size_t G_LOGGER_SIZE = 0; 145 | ib_msg_log_t G_LOGGER_FN = (ib_msg_log_t) raw_logger; 146 | static FILE* G_LOGGER_FH = NULL; 147 | 148 | /** 149 | * Column IDs 150 | */ 151 | #define KEY_COL 0 152 | #define VALUE_COL 1 153 | 154 | /** 155 | * Helper macros for txn and failure management 156 | */ 157 | #define FAIL_ON_ERROR(fn, block) \ 158 | if (fn != DB_SUCCESS) \ 159 | block 160 | 161 | #define ROLLBACK_RETURN(txn) { \ 162 | if (ib_trx_state(txn) != IB_TRX_NOT_STARTED) \ 163 | ib_trx_rollback(txn); \ 164 | else \ 165 | ib_trx_release(txn); \ 166 | return; } 167 | 168 | #define DRIVER_FREE_IF_NE(ptr, origin, origin_sz) { \ 169 | if (ptr < origin || ptr > (origin + origin_sz)) \ 170 | driver_free(ptr); } 171 | 172 | 173 | 174 | DRIVER_INIT(innostore_drv) 175 | { 176 | return &innostore_drv_entry; 177 | } 178 | 179 | static int innostore_drv_init() 180 | { 181 | char log_filename[_POSIX_PATH_MAX]; 182 | size_t log_filename_size = sizeof(log_filename); 183 | ErlDrvSysInfo sys_info; 184 | 185 | G_ENGINE_STATE_LOCK = erl_drv_mutex_create("innostore_state_lock"); 186 | G_LOGGER_LOCK = erl_drv_mutex_create("innostore_logger_lock"); 187 | 188 | // Check if this is beam.smp - cannot run under beam 189 | // due to restrictions with driver_send_term 190 | driver_system_info(&sys_info, sizeof(sys_info)); 191 | if (sys_info.smp_support == 0) 192 | { 193 | log("Innostore only supports the SMP runtime, add -smp enable"); 194 | return -1; 195 | } 196 | 197 | // Initialize Inno's memory subsystem 198 | if (ib_init() != DB_SUCCESS) 199 | { 200 | return -1; 201 | } 202 | 203 | // Set up the logger 204 | if (erl_drv_getenv("INNOSTORE_LOG", log_filename, &log_filename_size) == 0) 205 | { 206 | set_log_file(log_filename); 207 | } 208 | else 209 | { 210 | set_log_file(NULL); 211 | } 212 | 213 | return 0; 214 | } 215 | 216 | static void innostore_drv_finish() 217 | { 218 | erl_drv_mutex_destroy(G_ENGINE_STATE_LOCK); 219 | 220 | // Shutdown the engine, if it's running -- note that this blocks the 221 | // the calling VM thread and may be a long running operation. 222 | if (G_ENGINE_STATE == ENGINE_STARTED) 223 | { 224 | ib_err_t result = ib_shutdown(IB_SHUTDOWN_NORMAL); 225 | if (result != DB_SUCCESS) 226 | { 227 | log("ib_shutdown failed: %s\n", ib_strerror(result)); 228 | } 229 | } 230 | 231 | // Clean up logging after inno has shutdown completely 232 | erl_drv_mutex_destroy(G_LOGGER_LOCK); 233 | if (G_LOGGER_BUF != NULL) 234 | { 235 | driver_free(G_LOGGER_BUF); 236 | G_LOGGER_BUF = NULL; 237 | G_LOGGER_SIZE = 0; 238 | } 239 | if (G_LOGGER_FH != NULL) 240 | { 241 | fclose(G_LOGGER_FH); 242 | G_LOGGER_FH = NULL; 243 | G_LOGGER_FN = raw_logger; 244 | } 245 | 246 | G_ENGINE_STATE = ENGINE_STOPPED; 247 | } 248 | 249 | static ErlDrvData innostore_drv_start(ErlDrvPort port, char* buffer) 250 | { 251 | PortState* state = (PortState*)driver_alloc(sizeof(PortState)); 252 | int worker_rc; 253 | 254 | memset(state, '\0', sizeof(PortState)); 255 | 256 | // Save handle to the port 257 | state->port = port; 258 | 259 | // Save the owner PID 260 | state->port_owner = driver_connected(port); 261 | 262 | // Initialize in the READY state 263 | state->port_state = STATE_READY; 264 | 265 | // Make sure port is running in binary mode 266 | set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY); 267 | 268 | // Allocate a mutex and condition variable for the worker 269 | state->worker_lock = erl_drv_mutex_create("innostore_worker_lock"); 270 | state->worker_cv = erl_drv_cond_create("innostore_worker_cv"); 271 | 272 | // Spin up the worker 273 | worker_rc = erl_drv_thread_create("innostore_worker", &(state->worker), 274 | &innostore_worker, state, 0); 275 | if (state->worker_lock != NULL && 276 | state->worker_cv != NULL && 277 | worker_rc == 0) 278 | { 279 | return (ErlDrvData)state; 280 | } 281 | else 282 | { 283 | log("Innostore: Could not create port [lock=%p, cv=%p]\n", 284 | state->worker_lock, state->worker_cv); 285 | 286 | if (state->worker_cv != NULL) 287 | erl_drv_cond_destroy(state->worker_cv); 288 | 289 | if (state->worker_lock != NULL) 290 | erl_drv_mutex_destroy(state->worker_lock); 291 | 292 | driver_free(state); 293 | 294 | errno = worker_rc; 295 | return (ErlDrvData) ERL_DRV_ERROR_ERRNO; 296 | } 297 | } 298 | 299 | static void innostore_drv_stop(ErlDrvData handle) 300 | { 301 | PortState* state = (PortState*)handle; 302 | 303 | // Grab the worker lock, in case we have an job running 304 | erl_drv_mutex_lock(state->worker_lock); 305 | 306 | // Signal the shutdown and wait until the current operation has completed 307 | state->shutdown_flag = 1; 308 | erl_drv_cond_signal(state->worker_cv); 309 | 310 | while (state->op) 311 | { 312 | erl_drv_cond_wait(state->worker_cv, state->worker_lock); 313 | } 314 | 315 | // If the port state is not marked as READY, close the cursor and abort the txn 316 | if (state->port_state != STATE_READY) 317 | { 318 | ib_cursor_close(state->cursor); 319 | ib_trx_rollback(state->txn); 320 | } 321 | 322 | // No pending jobs and we have the lock again -- join our worker thread 323 | erl_drv_cond_signal(state->worker_cv); 324 | erl_drv_mutex_unlock(state->worker_lock); 325 | erl_drv_thread_join(state->worker, 0); 326 | 327 | // Cleanup 328 | erl_drv_cond_destroy(state->worker_cv); 329 | erl_drv_mutex_destroy(state->worker_lock); 330 | driver_free(handle); 331 | } 332 | 333 | 334 | #define is_cmd(bitmask) ((cmd & (bitmask)) == cmd) 335 | 336 | static int innostore_drv_control(ErlDrvData handle, unsigned int cmd, 337 | char* inbuf, int inbuf_sz, 338 | char** outbuf, int outbuf_sz) 339 | { 340 | PortState* state = (PortState*)handle; 341 | 342 | // Grab the worker lock. 343 | erl_drv_mutex_lock(state->worker_lock); 344 | 345 | assert(state->op == 0); 346 | 347 | // Verify that caller is not attempting cursor operation when no cursor is 348 | // active (or vice-versa) 349 | if (state->port_state == STATE_READY && is_cmd(CMD_CURSOR_OPS)) 350 | { 351 | erl_drv_mutex_unlock(state->worker_lock); 352 | send_error_atom(state, "cursor_not_open"); 353 | return 0; 354 | } 355 | else if (state->port_state == STATE_CURSOR && !is_cmd(CMD_CURSOR_OPS)) 356 | { 357 | erl_drv_mutex_unlock(state->worker_lock); 358 | send_error_atom(state, "cursor_is_open"); 359 | return 0; 360 | } 361 | 362 | // Copy inbuf into our work buffer 363 | if (inbuf_sz > 0) 364 | { 365 | state->work_buffer = driver_alloc(inbuf_sz); 366 | memcpy(state->work_buffer, inbuf, inbuf_sz); 367 | } 368 | 369 | // Select the appropriate async op 370 | switch (cmd) 371 | { 372 | case CMD_SET_CFG: 373 | state->op = &do_set_cfg; 374 | break; 375 | 376 | case CMD_START: 377 | state->op = &do_start; 378 | break; 379 | 380 | case CMD_INIT_TABLE: 381 | state->op = &do_init_table; 382 | break; 383 | 384 | case CMD_IS_STARTED: 385 | // Check the global state -- prior to doing that, release our worker lock 386 | // so as to avoid weird locking overlaps. Store the true/false value as 387 | // a single byte in outbuf. No need to do any allocation as the VM always 388 | // provides a 64-byte outbuf buffer by default. 389 | assert(outbuf_sz > 0); 390 | erl_drv_mutex_unlock(state->worker_lock); 391 | erl_drv_mutex_lock(G_ENGINE_STATE_LOCK); 392 | (*outbuf)[0] = (G_ENGINE_STATE == ENGINE_STARTED); 393 | erl_drv_mutex_unlock(G_ENGINE_STATE_LOCK); 394 | return 1; 395 | 396 | case CMD_GET: 397 | state->op = &do_get; 398 | break; 399 | 400 | case CMD_PUT: 401 | state->op = &do_put; 402 | break; 403 | 404 | case CMD_DELETE: 405 | state->op = &do_delete; 406 | break; 407 | 408 | case CMD_LIST_TABLES: 409 | state->op = &do_list_tables; 410 | break; 411 | 412 | case CMD_CURSOR_OPEN: 413 | state->op = &do_cursor_open; 414 | break; 415 | 416 | case CMD_CURSOR_MOVE: 417 | state->op = &do_cursor_move; 418 | break; 419 | 420 | case CMD_CURSOR_CLOSE: 421 | state->op = &do_cursor_close; 422 | break; 423 | 424 | case CMD_DROP_TABLE: 425 | state->op = &do_drop_table; 426 | break; 427 | 428 | case CMD_STATUS: 429 | state->op = &do_status; 430 | break; 431 | } 432 | 433 | // Signal the worker 434 | erl_drv_cond_signal(state->worker_cv); 435 | erl_drv_mutex_unlock(state->worker_lock); 436 | *outbuf = 0; 437 | return 0; 438 | } 439 | 440 | 441 | static void* innostore_worker(void* arg) 442 | { 443 | PortState* state = (PortState*)arg; 444 | erl_drv_mutex_lock(state->worker_lock); 445 | while (1) 446 | { 447 | // 448 | // NOTE: Holds the worker lock for the duration of the loop !! 449 | // 450 | if (state->shutdown_flag) 451 | { 452 | driver_free(state->work_buffer); 453 | state->work_buffer = 0; 454 | if (state->op != 0) 455 | { 456 | send_error_atom(state, "stopping"); 457 | state->op = 0; 458 | } 459 | erl_drv_cond_signal(state->worker_cv); 460 | erl_drv_mutex_unlock(state->worker_lock); 461 | break; 462 | } 463 | 464 | if (state->op) 465 | { 466 | state->op(state); 467 | state->op = 0; 468 | driver_free(state->work_buffer); 469 | state->work_buffer = 0; 470 | } 471 | else 472 | { 473 | erl_drv_cond_wait(state->worker_cv, state->worker_lock); 474 | } 475 | } 476 | return 0; 477 | } 478 | 479 | static void do_set_cfg(void* arg) 480 | { 481 | PortState* state = (PortState*)arg; 482 | 483 | erl_drv_mutex_lock(G_ENGINE_STATE_LOCK); 484 | if (G_ENGINE_STATE == ENGINE_STOPPED) 485 | { 486 | char* key = UNPACK_STRING(state->work_buffer, 0); 487 | const char* value = UNPACK_STRING(state->work_buffer, strlen(key)+1); 488 | 489 | if (strcmp(key, "error_log") == 0) 490 | { 491 | if (set_log_file(value) == 0) 492 | { 493 | send_ok(state); 494 | } 495 | else 496 | { 497 | send_error_atom(state, "einval"); 498 | } 499 | } 500 | else 501 | { 502 | // Check the expected type of the provided key so as to 1. validate it's a good key 503 | // and 2. know what setter to use. 504 | ib_cfg_type_t key_type; 505 | ib_err_t error = ib_cfg_var_get_type(key, &key_type); 506 | if (error == DB_SUCCESS) 507 | { 508 | if (key_type == IB_CFG_TEXT) 509 | { 510 | // HACK: Semantics of setting a text configuration value for innodb changed 511 | // to be pointer assignment (from copy) for vsn 1.0.6.6750. So, we strdup the 512 | // value to ensure everything works as expected. 513 | // TODO: Setup some sort of list of strdup'd values to ensure they all get 514 | // cleaned up properly. In typical usage, this isn't going to be a problem 515 | // as you only initialize once per run, but it bothers me just the same. 516 | error = ib_cfg_set(key, strdup(value)); 517 | } 518 | else 519 | { 520 | ErlDrvUInt value_i; 521 | UNPACK_INT(state->work_buffer, strlen(key)+1, &value_i); 522 | error = ib_cfg_set(key, value_i); 523 | } 524 | 525 | } 526 | 527 | if (error == DB_SUCCESS) 528 | { 529 | send_ok(state); 530 | } 531 | else 532 | { 533 | send_error_str(state, ib_strerror(error)); 534 | } 535 | } 536 | } 537 | else 538 | { 539 | send_error_atom(state, "starting"); 540 | } 541 | 542 | erl_drv_mutex_unlock(G_ENGINE_STATE_LOCK); 543 | } 544 | 545 | static void do_start(void* arg) 546 | { 547 | PortState* state = (PortState*)arg; 548 | 549 | erl_drv_mutex_lock(G_ENGINE_STATE_LOCK); 550 | if (G_ENGINE_STATE == ENGINE_STOPPED) 551 | { 552 | // Engine was stopped; set the flag, unlock and run the start 553 | G_ENGINE_STATE = ENGINE_STARTING; 554 | erl_drv_mutex_unlock(G_ENGINE_STATE_LOCK); 555 | 556 | // Run the startup without holding any lock -- this can take a while 557 | // if we are recovering from previous errors 558 | ib_err_t error = ib_startup("barracuda"); 559 | 560 | if (error == DB_SUCCESS) 561 | { 562 | // Make sure the innokeystore database exists 563 | // TODO: Avoid hard-coding the db name here... 564 | if (ib_database_create("innokeystore") != IB_TRUE) 565 | { 566 | error = DB_ERROR; 567 | } 568 | } 569 | 570 | // Relock and sort out results 571 | erl_drv_mutex_lock(G_ENGINE_STATE_LOCK); 572 | if (error == DB_SUCCESS) 573 | { 574 | G_ENGINE_STATE = ENGINE_STARTED; 575 | send_ok(state); 576 | } 577 | else 578 | { 579 | G_ENGINE_STATE = ENGINE_STOPPED; 580 | send_error_str(state, ib_strerror(error)); 581 | } 582 | } 583 | else if (G_ENGINE_STATE == ENGINE_STARTED) 584 | { 585 | // another thread has already completed startup 586 | send_ok(state); 587 | } 588 | else 589 | { 590 | // Engine was not in stopped state when do_start was called. 591 | // Probably due to multiple threads trying to start at the 592 | // same time. 593 | assert(G_ENGINE_STATE == ENGINE_STARTING); 594 | send_error_atom(state, "starting"); 595 | } 596 | 597 | erl_drv_mutex_unlock(G_ENGINE_STATE_LOCK); 598 | } 599 | 600 | static void do_init_table(void* arg) 601 | { 602 | PortState* state = (PortState*)arg; 603 | 604 | unsigned char fmt_code = UNPACK_BYTE(state->work_buffer, 0); 605 | // Unpack the table name (pre-formatted to be Database/TableName) 606 | char* table = UNPACK_STRING(state->work_buffer, 1); 607 | ib_id_t table_id; 608 | ib_tbl_fmt_t table_fmt; 609 | 610 | switch(fmt_code) 611 | { 612 | case FORMAT_REDUNDANT: 613 | table_fmt = IB_TBL_REDUNDANT; 614 | break; 615 | case FORMAT_COMPACT: 616 | table_fmt = IB_TBL_COMPACT; 617 | break; 618 | case FORMAT_DYNAMIC: 619 | table_fmt = IB_TBL_DYNAMIC; 620 | break; 621 | case FORMAT_COMPRESSED: 622 | table_fmt = IB_TBL_COMPRESSED; 623 | break; 624 | default: 625 | send_error_atom(state, "bad_format"); 626 | return; 627 | } 628 | 629 | // If the table doesn't exist, create it 630 | if (ib_table_get_id(table, &table_id) != DB_SUCCESS) 631 | { 632 | // Start a txn for schema access and be sure to make it serializable 633 | ib_trx_t txn = ib_trx_begin(IB_TRX_SERIALIZABLE); 634 | ib_schema_lock_exclusive(txn); 635 | 636 | // Create the table schema 637 | ib_tbl_sch_t schema; 638 | ib_table_schema_create(table, &schema, table_fmt, 0); 639 | ib_table_schema_add_col(schema, "key", IB_VARBINARY, IB_COL_NONE, 0, 255); 640 | ib_table_schema_add_col(schema, "value", IB_BLOB, IB_COL_NONE, 0, 0); 641 | 642 | // Create primary index on key 643 | ib_idx_sch_t index; 644 | ib_table_schema_add_index(schema, "PRIMARY_KEY", &index); 645 | ib_index_schema_add_col(index, "key", 0); 646 | ib_index_schema_set_clustered(index); 647 | 648 | // Create the actual table 649 | ib_err_t rc = ib_table_create(txn, schema, &table_id); 650 | 651 | // Release the schema -- doesn't matter if table was created or not at this point 652 | ib_schema_unlock(txn); 653 | 654 | if (rc == DB_SUCCESS) 655 | { 656 | // Commit changes to schema (if any) 657 | ib_trx_commit(txn); 658 | } 659 | else 660 | { 661 | // Failed to create table -- rollback and exit 662 | ib_trx_rollback(txn); 663 | send_error_str(state, ib_strerror(rc)); 664 | return; 665 | } 666 | } 667 | 668 | // Guaranteed at this point to have a valid table_id 669 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 670 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)&table_id, 671 | (ErlDrvUInt)sizeof(table_id), 672 | ERL_DRV_TUPLE, 2}; 673 | driver_send_term(state->port, state->port_owner, response, 674 | sizeof(response) / sizeof(response[0])); 675 | } 676 | 677 | static void do_get(void* arg) 678 | { 679 | PortState* state = (PortState*)arg; 680 | 681 | // Unpack key from work buffer 682 | // table - 8 bytes 683 | // keysz - 1 byte 684 | // key - variable 685 | ib_id_t table ; UNPACK_INT(state->work_buffer, 0, &table); 686 | unsigned char keysz = UNPACK_BYTE(state->work_buffer, sizeof(table)); 687 | char* key = UNPACK_BLOB(state->work_buffer, sizeof(table)+1); 688 | 689 | ib_trx_t txn = ib_trx_begin(IB_TRX_REPEATABLE_READ); 690 | 691 | ib_crsr_t cursor; 692 | FAIL_ON_ERROR(ib_cursor_open_table_using_id(table, txn, &cursor), 693 | ROLLBACK_RETURN(txn)); 694 | 695 | ib_tpl_t key_tuple = ib_clust_search_tuple_create(cursor); 696 | ib_col_set_value(key_tuple, KEY_COL, key, keysz); 697 | 698 | int searchloc; 699 | ib_err_t error = ib_cursor_moveto(cursor, key_tuple, IB_CUR_GE, &searchloc); 700 | 701 | // Drop the key tuple -- cursor is at desired location 702 | ib_tuple_delete(key_tuple); 703 | 704 | // If we encountered an error, bail 705 | if (error != DB_SUCCESS && error != DB_RECORD_NOT_FOUND && error != DB_END_OF_INDEX) 706 | { 707 | ib_cursor_close(cursor); 708 | send_error_str(state, ib_strerror(error)); 709 | ROLLBACK_RETURN(txn); 710 | } 711 | 712 | // Found it, read the value and send back to caller 713 | if (searchloc == 0) 714 | { 715 | ib_tpl_t tuple = ib_clust_read_tuple_create(cursor); 716 | 717 | // TODO: May need better error handling here 718 | FAIL_ON_ERROR(ib_cursor_read_row(cursor, tuple), 719 | { 720 | ib_tuple_delete(tuple); 721 | ib_cursor_close(cursor); 722 | ROLLBACK_RETURN(txn); 723 | }); 724 | 725 | // Get the size of the value 726 | ib_col_meta_t value_meta; 727 | int raw_value_sz = ib_col_get_meta(tuple, VALUE_COL, &value_meta); 728 | char* raw_value = (char*)ib_col_get_value(tuple, VALUE_COL); 729 | 730 | unsigned int value_sz = 0; 731 | char* value = 0; 732 | 733 | // Decompress the raw value as necessary. If it fails, that layer 734 | // generates an error -- we just need to cleanup 735 | if (decompress(state, raw_value, raw_value_sz, &value, &value_sz)) 736 | { 737 | ib_tuple_delete(tuple); 738 | ib_cursor_close(cursor); 739 | ROLLBACK_RETURN(txn); 740 | } 741 | 742 | // Send back the response 743 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 744 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)value, 745 | (ErlDrvUInt)value_sz, 746 | ERL_DRV_TUPLE, 2}; 747 | driver_send_term(state->port, state->port_owner, 748 | response, sizeof(response) / sizeof(response[0])); 749 | 750 | // Cleanup 751 | DRIVER_FREE_IF_NE(value, raw_value, raw_value_sz); 752 | ib_tuple_delete(tuple); 753 | } 754 | else 755 | { 756 | // Not found 757 | send_ok_atom(state, "not_found"); 758 | } 759 | 760 | ib_cursor_close(cursor); 761 | ib_trx_commit(txn); 762 | } 763 | 764 | static void do_put(void* arg) 765 | { 766 | PortState* state = (PortState*)arg; 767 | 768 | // Unpack key/value from work buffer: 769 | // table - 8 bytes 770 | // cflag - 1 byte (compression flag) 771 | // keysz - 1 byte 772 | // key - variable 773 | // valuesz - 4 bytes 774 | // value - variable 775 | ib_id_t table ; UNPACK_INT(state->work_buffer, 0, &table); 776 | unsigned char cflag = UNPACK_BYTE(state->work_buffer, sizeof(table)); 777 | unsigned char keysz = UNPACK_BYTE(state->work_buffer, sizeof(table) + 1); 778 | char* key = UNPACK_BLOB(state->work_buffer, sizeof(table) + 2); 779 | unsigned int raw_valuesz; UNPACK_INT(state->work_buffer, sizeof(table) + 2 + keysz, &raw_valuesz); 780 | char* raw_value = UNPACK_BLOB(state->work_buffer, sizeof(table) + 2 + keysz + 4); 781 | 782 | unsigned int valuesz; 783 | char* value; 784 | 785 | // Compress the data as requested -- returns non-zero if compression failed. Also, 786 | // the underlying compression mechanism/function is responsible for sending an error 787 | // message in that situation. 788 | if (compress(cflag, state, raw_value, raw_valuesz, &value, &valuesz)) 789 | { 790 | return; 791 | } 792 | 793 | ib_trx_t txn = ib_trx_begin(IB_TRX_SERIALIZABLE); 794 | 795 | ib_crsr_t cursor; 796 | FAIL_ON_ERROR(ib_cursor_open_table_using_id(table, txn, &cursor), 797 | { 798 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 799 | ROLLBACK_RETURN(txn); 800 | }); 801 | 802 | // Lock the cursor for exclusive access to our row 803 | FAIL_ON_ERROR(ib_cursor_set_lock_mode(cursor, IB_LOCK_X), 804 | { 805 | ib_cursor_close(cursor); 806 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 807 | ROLLBACK_RETURN(txn); 808 | }); 809 | 810 | // Setup the search tuple and copy our key into it -- then move to the desired 811 | // location 812 | int searchloc; 813 | ib_tpl_t key_tuple = ib_clust_search_tuple_create(cursor); 814 | ib_col_set_value(key_tuple, KEY_COL, key, keysz); 815 | ib_err_t error = ib_cursor_moveto(cursor, key_tuple, IB_CUR_GE, &searchloc); 816 | 817 | // Drop the key tuple -- no longer important 818 | ib_tuple_delete(key_tuple); 819 | 820 | // If we didn't find it or reach the end of the index looking for it, fail 821 | if (error != DB_SUCCESS && error != DB_END_OF_INDEX) 822 | { 823 | ib_cursor_close(cursor); 824 | send_error_str(state, ib_strerror(error)); 825 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 826 | ROLLBACK_RETURN(txn); 827 | } 828 | 829 | // Exact match, update it 830 | if (searchloc == 0) 831 | { 832 | ib_tpl_t old = ib_clust_read_tuple_create(cursor); 833 | ib_tpl_t new = ib_clust_read_tuple_create(cursor); 834 | 835 | FAIL_ON_ERROR(ib_cursor_read_row(cursor, old), 836 | { 837 | ib_tuple_delete(old); 838 | ib_tuple_delete(new); 839 | ib_cursor_close(cursor); 840 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 841 | ROLLBACK_RETURN(txn); 842 | }); 843 | 844 | ib_tuple_copy(new, old); 845 | ib_col_set_value(new, VALUE_COL, value, valuesz); 846 | 847 | FAIL_ON_ERROR(ib_cursor_update_row(cursor, old, new), 848 | { 849 | ib_tuple_delete(old); 850 | ib_tuple_delete(new); 851 | ib_cursor_close(cursor); 852 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 853 | ROLLBACK_RETURN(txn); 854 | }); 855 | 856 | ib_tuple_delete(old); 857 | ib_tuple_delete(new); 858 | } 859 | else 860 | { 861 | // No match -- insert a new row 862 | ib_tpl_t new = ib_clust_read_tuple_create(cursor); 863 | ib_col_set_value(new, KEY_COL, key, keysz); 864 | ib_col_set_value(new, VALUE_COL, value, valuesz); 865 | 866 | FAIL_ON_ERROR(ib_cursor_insert_row(cursor, new), 867 | { 868 | ib_tuple_delete(new); 869 | ib_cursor_close(cursor); 870 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 871 | ROLLBACK_RETURN(txn); 872 | }); 873 | 874 | ib_tuple_delete(new); 875 | } 876 | 877 | // All done -- cleanup cursor and txn 878 | ib_cursor_close(cursor); 879 | DRIVER_FREE_IF_NE(value, raw_value, raw_valuesz); 880 | ib_trx_commit(txn); 881 | send_ok(state); 882 | } 883 | 884 | static void do_delete(void* arg) 885 | { 886 | PortState* state = (PortState*)arg; 887 | 888 | // Unpack key/value from work buffer: 889 | // table - 8 bytes 890 | // keysz - 1 byte 891 | // key - variable 892 | ib_id_t table ; UNPACK_INT(state->work_buffer, 0, &table); 893 | unsigned char keysz = UNPACK_BYTE(state->work_buffer, sizeof(table)); 894 | char* key = UNPACK_BLOB(state->work_buffer, sizeof(table) + 1); 895 | 896 | ib_trx_t txn = ib_trx_begin(IB_TRX_SERIALIZABLE); 897 | 898 | ib_crsr_t cursor; 899 | FAIL_ON_ERROR(ib_cursor_open_table_using_id(table, txn, &cursor), 900 | ROLLBACK_RETURN(txn)); 901 | 902 | // Lock the cursor for exclusive access to our row 903 | FAIL_ON_ERROR(ib_cursor_set_lock_mode(cursor, IB_LOCK_X), 904 | { 905 | ib_cursor_close(cursor); 906 | ROLLBACK_RETURN(txn); 907 | }); 908 | 909 | // Setup the search tuple and copy our key into it -- then move to the desired 910 | // location 911 | int searchloc; 912 | ib_tpl_t key_tuple = ib_clust_search_tuple_create(cursor); 913 | ib_col_set_value(key_tuple, KEY_COL, key, keysz); 914 | ib_err_t error = ib_cursor_moveto(cursor, key_tuple, IB_CUR_GE, &searchloc); 915 | 916 | // Drop the key tuple -- no longer important 917 | ib_tuple_delete(key_tuple); 918 | 919 | // If we encountered an error, bail 920 | if (error != DB_SUCCESS && error != DB_END_OF_INDEX) 921 | { 922 | ib_cursor_close(cursor); 923 | send_error_str(state, ib_strerror(error)); 924 | ROLLBACK_RETURN(txn); 925 | } 926 | 927 | // Exact match, delete it. If it wasn't found noop. 928 | if (searchloc == 0) 929 | { 930 | FAIL_ON_ERROR(ib_cursor_delete_row(cursor), 931 | { 932 | ib_cursor_close(cursor); 933 | ROLLBACK_RETURN(txn); 934 | }); 935 | } 936 | 937 | // All done -- cleanup cursor and txn and notify caller 938 | ib_cursor_close(cursor); 939 | ib_trx_commit(txn); 940 | send_ok(state); 941 | } 942 | 943 | static void do_list_tables(void* arg) 944 | { 945 | PortState* state = (PortState*)arg; 946 | 947 | // Spin up a transaction and lock the schema. Then use the iteration visitor 948 | // function to stream the list of names back out to the caller. 949 | ib_trx_t txn = ib_trx_begin(IB_TRX_SERIALIZABLE); 950 | 951 | FAIL_ON_ERROR(ib_schema_lock_exclusive(txn), 952 | ROLLBACK_RETURN(txn)); 953 | 954 | ib_err_t error = ib_schema_tables_iterate(txn, &do_list_tables_cb, state); 955 | if (error == DB_SUCCESS) 956 | { 957 | send_ok(state); 958 | } 959 | else 960 | { 961 | send_error_str(state, ib_strerror(error)); 962 | } 963 | 964 | ib_schema_unlock(txn); 965 | ib_trx_commit(txn); 966 | } 967 | 968 | static int do_list_tables_cb(void* arg, const char* tablename, int tablename_sz) 969 | { 970 | PortState* state = (PortState*)arg; 971 | 972 | // Only send tables that are not prefixed w/ SYS_ 973 | if (strncmp("SYS_", tablename, 4) != 0) 974 | { 975 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_table_name"), 976 | ERL_DRV_STRING, (ErlDrvTermData)tablename, (ErlDrvUInt)tablename_sz, 977 | ERL_DRV_TUPLE, 2}; 978 | driver_send_term(state->port, state->port_owner, response, 979 | sizeof(response) / sizeof(response[0])); 980 | } 981 | return 0; 982 | } 983 | 984 | static void do_cursor_open(void* arg) 985 | { 986 | PortState* state = (PortState*)arg; 987 | 988 | // Unpack the table ID to scan 989 | ib_id_t table ; UNPACK_INT(state->work_buffer, 0, &table); 990 | 991 | // Start the txn and open a cursor for the table. 992 | state->txn = ib_trx_begin(IB_TRX_READ_UNCOMMITTED); 993 | FAIL_ON_ERROR(ib_cursor_open_table_using_id(table, state->txn, &(state->cursor)), 994 | ROLLBACK_RETURN(state->txn)); 995 | 996 | // Update port state 997 | state->port_state = STATE_CURSOR; 998 | 999 | // Notify caller 1000 | send_ok(state); 1001 | } 1002 | 1003 | static void do_cursor_move(void* arg) 1004 | { 1005 | PortState* state = (PortState*)arg; 1006 | 1007 | // Unpack the type of move, and what data to return (key or key+data) 1008 | // movement - 1 byte 1009 | // content - 1 byte 1010 | unsigned char movement = UNPACK_BYTE(state->work_buffer, 0); 1011 | unsigned char content = UNPACK_BYTE(state->work_buffer, 1); 1012 | 1013 | ib_err_t error; 1014 | switch(movement) 1015 | { 1016 | case CURSOR_FIRST: error = ib_cursor_first(state->cursor); break; 1017 | case CURSOR_NEXT : error = ib_cursor_next(state->cursor); break; 1018 | case CURSOR_PREV : error = ib_cursor_prev(state->cursor); break; 1019 | case CURSOR_LAST : error = ib_cursor_last(state->cursor); break; 1020 | } 1021 | 1022 | if (error == DB_SUCCESS) 1023 | { 1024 | do_cursor_read(content, state); 1025 | } 1026 | else if (error == DB_END_OF_INDEX) 1027 | { 1028 | send_ok_atom(state, "eof"); 1029 | } 1030 | else 1031 | { 1032 | send_error_str(state, ib_strerror(error)); 1033 | ib_cursor_close(state->cursor); 1034 | ib_trx_t txn = state->txn; 1035 | state->cursor = 0; 1036 | state->txn = 0; 1037 | state->port_state = STATE_READY; 1038 | ROLLBACK_RETURN(txn); 1039 | } 1040 | } 1041 | 1042 | static void do_cursor_read(unsigned int content_flag, PortState* state) 1043 | { 1044 | ib_tpl_t tuple = ib_clust_read_tuple_create(state->cursor); 1045 | ib_err_t error = ib_cursor_read_row(state->cursor, tuple); 1046 | if (error == DB_SUCCESS) 1047 | { 1048 | // Pull out the key 1049 | ib_col_meta_t key_col_meta; 1050 | int key_sz = ib_col_get_meta(tuple, KEY_COL, &key_col_meta); 1051 | char* key = (char*) ib_col_get_value(tuple, KEY_COL); 1052 | 1053 | // Only send back the key if the caller isn't interested in value 1054 | if (content_flag == CONTENT_KEY_ONLY) 1055 | { 1056 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 1057 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)key, (ErlDrvUInt)key_sz, 1058 | ERL_DRV_TUPLE, 2}; 1059 | driver_send_term(state->port, state->port_owner, response, 1060 | sizeof(response) / sizeof(response[0])); 1061 | } 1062 | else 1063 | { 1064 | // Caller wants key and values. Need to pull out the value, decompress it, etc. 1065 | ib_col_meta_t value_col_meta; 1066 | int raw_value_sz = ib_col_get_meta(tuple, VALUE_COL, &value_col_meta); 1067 | char* raw_value = (char*) ib_col_get_value(tuple, VALUE_COL); 1068 | 1069 | unsigned int value_sz = 0; 1070 | char* value = 0; 1071 | 1072 | // If decompress fails, that layer generates an error -- we just need to cleanup 1073 | if (decompress(state, raw_value, raw_value_sz, &value, &value_sz)) 1074 | { 1075 | ib_tuple_delete(tuple); 1076 | ib_cursor_close(state->cursor); 1077 | ib_trx_t txn = state->txn; 1078 | state->cursor = 0; 1079 | state->txn = 0; 1080 | state->port_state = STATE_READY; 1081 | ROLLBACK_RETURN(txn); 1082 | } 1083 | 1084 | // Send back the key and value 1085 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 1086 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)key, (ErlDrvUInt)key_sz, 1087 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)value, (ErlDrvUInt)value_sz, 1088 | ERL_DRV_TUPLE, 3}; 1089 | driver_send_term(state->port, state->port_owner, 1090 | response, sizeof(response) / sizeof(response[0])); 1091 | 1092 | // Cleanup 1093 | DRIVER_FREE_IF_NE(value, raw_value, raw_value_sz); 1094 | } 1095 | } 1096 | else 1097 | { 1098 | // Notify caller and then clean everything up -- tear down cursor, etc. 1099 | send_error_str(state, ib_strerror(error)); 1100 | ib_tuple_delete(tuple); 1101 | ib_cursor_close(state->cursor); 1102 | ib_trx_t txn = state->txn; 1103 | state->cursor = 0; 1104 | state->txn = 0; 1105 | state->port_state = STATE_READY; 1106 | ROLLBACK_RETURN(txn); 1107 | } 1108 | 1109 | // Cleanup the tuple 1110 | ib_tuple_delete(tuple); 1111 | } 1112 | 1113 | static void do_cursor_close(void* arg) 1114 | { 1115 | PortState* state = (PortState*)arg; 1116 | 1117 | // Cleanup the cursor and transaction and reset our state 1118 | ib_cursor_close(state->cursor); 1119 | ib_trx_commit(state->txn); 1120 | state->cursor = 0; 1121 | state->txn = 0; 1122 | state->port_state = STATE_READY; 1123 | 1124 | // Notify caller 1125 | send_ok(state); 1126 | } 1127 | 1128 | static void do_drop_table(void* arg) 1129 | { 1130 | PortState* state = (PortState*)arg; 1131 | 1132 | char* table = UNPACK_STRING(state->work_buffer, 0); 1133 | 1134 | // Use a serializable txn as this is a schema op. Also be sure to lock 1135 | // our schema exclusively, per docs. 1136 | ib_trx_t txn = ib_trx_begin(IB_TRX_SERIALIZABLE); 1137 | 1138 | ib_err_t rc = ib_schema_lock_exclusive(txn); 1139 | FAIL_ON_ERROR(rc, 1140 | { 1141 | send_error_str(state, ib_strerror(rc)); 1142 | ROLLBACK_RETURN(txn); 1143 | }); 1144 | 1145 | // Do the actual drop 1146 | rc = ib_table_drop(txn, table); 1147 | 1148 | // Go ahead and unlock the schema -- this has to be done no matter the outcome. 1149 | ib_schema_unlock(txn); 1150 | 1151 | // If everything went well, commit; otherwise rollback 1152 | if (rc == DB_SUCCESS) 1153 | { 1154 | ib_trx_commit(txn); 1155 | send_ok(state); 1156 | } 1157 | else 1158 | { 1159 | ib_trx_rollback(txn); 1160 | send_error_str(state, ib_strerror(rc)); 1161 | } 1162 | } 1163 | 1164 | static void do_status(void* arg) 1165 | { 1166 | PortState* state = (PortState*)arg; 1167 | char* var_name = UNPACK_STRING(state->work_buffer, 0); 1168 | ib_i64_t val; 1169 | 1170 | ib_err_t rc = ib_status_get_i64(var_name, &val); 1171 | if (rc == DB_SUCCESS) 1172 | { 1173 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 1174 | ERL_DRV_BUF2BINARY, (ErlDrvTermData)&val, 1175 | (ErlDrvUInt)sizeof(val), 1176 | ERL_DRV_TUPLE, 2}; 1177 | driver_send_term(state->port, state->port_owner, 1178 | response, sizeof(response) / sizeof(response[0])); 1179 | } 1180 | else 1181 | { 1182 | send_error_str(state, ib_strerror(rc)); 1183 | } 1184 | } 1185 | 1186 | 1187 | static int compress(unsigned int cflag, 1188 | PortState* state, char* in_value, unsigned int in_value_sz, 1189 | char** out_value, unsigned int* out_value_sz) 1190 | { 1191 | // Copy the buffer of data into the temporary space so we can prefix it 1192 | // with the compression marker byte 1193 | // TODO: Add support for other compression mechanisms 1194 | *out_value = driver_alloc(in_value_sz + 1); 1195 | memcpy((*out_value) + 1, in_value, in_value_sz); 1196 | (*out_value)[0] = COMPRESSION_NONE; 1197 | *out_value_sz = in_value_sz + 1; 1198 | return 0; 1199 | } 1200 | 1201 | static int decompress(PortState* state, char* in_value, unsigned int in_value_sz, 1202 | char** out_value, unsigned int* out_value_sz) 1203 | { 1204 | if (in_value[0] == COMPRESSION_NONE) 1205 | { 1206 | // Skip the compression header (1 byte) and return a pointer into in_value. The 1207 | // cleanup macro DRIVER_FREE_IF_NE deals with this and ensures we don't free 1208 | // the same chunk of memory twice 1209 | *out_value = in_value + 1; 1210 | *out_value_sz = in_value_sz - 1; 1211 | return 0; 1212 | } 1213 | else 1214 | { 1215 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_error"), 1216 | ERL_DRV_ATOM, driver_mk_atom("unknown_compression_method"), 1217 | ERL_DRV_INT, (ErlDrvUInt)in_value[0], 1218 | ERL_DRV_TUPLE, 2, 1219 | ERL_DRV_TUPLE, 2}; 1220 | driver_send_term(state->port, state->port_owner, response, 1221 | sizeof(response) / sizeof(response[0])); 1222 | return -1; 1223 | } 1224 | } 1225 | 1226 | static int set_log_file(const char* filename) 1227 | { 1228 | int ret = 0; 1229 | int open_errno = 0; 1230 | 1231 | erl_drv_mutex_lock(G_LOGGER_LOCK); 1232 | 1233 | /* Close any open log file */ 1234 | if (G_LOGGER_FH != NULL && G_LOGGER_FH != stderr) 1235 | { 1236 | fclose(G_LOGGER_FH); 1237 | G_LOGGER_FH = NULL; 1238 | } 1239 | 1240 | G_LOGGER_FN = (ib_msg_log_t) fprintf; 1241 | 1242 | /* If filename is non-NULL and non-empty, log to that file 1243 | */ 1244 | if (filename != NULL && *filename != '\0') 1245 | { 1246 | G_LOGGER_FH = fopen(filename, "a"); 1247 | if (G_LOGGER_FH == NULL) 1248 | { 1249 | open_errno = errno; 1250 | ret = -1; 1251 | } 1252 | else 1253 | { 1254 | setvbuf(G_LOGGER_FH, NULL, _IONBF, 0); 1255 | } 1256 | } 1257 | 1258 | /* If no log file given or open failed, log on stderr. Need to decide if the terminal 1259 | ** is in raw mode or not. If in raw mode need to output \r\n at the end 1260 | ** of lines instead of just \r 1261 | **/ 1262 | if (G_LOGGER_FH == NULL) 1263 | { 1264 | struct termios termios; 1265 | 1266 | G_LOGGER_FH = stderr; 1267 | if (tcgetattr(fileno(stderr), &termios) == 0) 1268 | { 1269 | if ((termios.c_oflag & OPOST) == 0) 1270 | { 1271 | G_LOGGER_FN = (ib_msg_log_t) raw_logger; 1272 | } 1273 | } 1274 | } 1275 | 1276 | /* Finally, let InnoDb know the logging function and file handle */ 1277 | ib_logger_set(G_LOGGER_FN, G_LOGGER_FH); 1278 | 1279 | erl_drv_mutex_unlock(G_LOGGER_LOCK); 1280 | 1281 | /* If a filename was passed in above but could not be opened, log 1282 | ** the error so it will be formatted nicely on the erlang console 1283 | */ 1284 | if (open_errno != 0) 1285 | { 1286 | log("Innostore: Could not open log file \"%s\" - %s\n", 1287 | filename, strerror(open_errno)); 1288 | log("Innostore: Logging to stderr\n"); 1289 | } 1290 | 1291 | 1292 | return ret; 1293 | } 1294 | 1295 | 1296 | /* Raw tty mode logger - convert all \n to \r\n. This is often called multiple times per log 1297 | ** line - once for the timestamp and again for the message. 1298 | ** 1299 | ** No error checking on i/o calls - if stderr is gone theres not much else to do 1300 | */ 1301 | static int raw_logger(ib_msg_stream_t stream, const char*fmt, ...) 1302 | { 1303 | int len; 1304 | va_list ap; 1305 | int done; 1306 | char *ptr; 1307 | char *eol; 1308 | 1309 | erl_drv_mutex_lock(G_LOGGER_LOCK); 1310 | 1311 | // Resize the log buffer until the message fits. 1312 | va_start(ap, fmt); 1313 | do 1314 | { 1315 | done = 1; 1316 | len = vsnprintf(G_LOGGER_BUF, G_LOGGER_SIZE, fmt, ap); 1317 | if (len >= G_LOGGER_SIZE) 1318 | { 1319 | G_LOGGER_SIZE = len + 128; 1320 | G_LOGGER_BUF = driver_realloc(G_LOGGER_BUF, G_LOGGER_SIZE); 1321 | done = 0; 1322 | } 1323 | } while(!done); 1324 | va_end(ap); 1325 | 1326 | // Scan through the log message and break on \n 1327 | ptr = G_LOGGER_BUF; 1328 | eol = ptr - 1; // catch messages starting with\n 1329 | while (ptr != NULL && *ptr != '\0') 1330 | { 1331 | eol = strchr(eol+1, '\n'); 1332 | if (eol == NULL) 1333 | { 1334 | fputs(ptr, stderr); 1335 | ptr = NULL; 1336 | } 1337 | else 1338 | { 1339 | size_t len = eol - ptr; // length of chars up to next LF 1340 | if (len > 0) 1341 | { 1342 | fwrite(ptr, len, 1, stderr); 1343 | } 1344 | fputc('\r', stderr); 1345 | ptr = eol; // ptr starts with next \n 1346 | } 1347 | } 1348 | 1349 | erl_drv_mutex_unlock(G_LOGGER_LOCK); 1350 | 1351 | return len; 1352 | } 1353 | 1354 | 1355 | static void send_ok(PortState* state) 1356 | { 1357 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok") }; 1358 | driver_send_term(state->port, state->port_owner, response, 1359 | sizeof(response) / sizeof(response[0])); 1360 | } 1361 | 1362 | static void send_ok_atom(PortState* state, const char* atom) 1363 | { 1364 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_ok"), 1365 | ERL_DRV_ATOM, driver_mk_atom((char*)atom), 1366 | ERL_DRV_TUPLE, 2}; 1367 | driver_send_term(state->port, state->port_owner, 1368 | response, sizeof(response) / sizeof(response[0])); 1369 | } 1370 | 1371 | static void send_error_atom(PortState* state, const char* atom) 1372 | { 1373 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_error"), 1374 | ERL_DRV_ATOM, driver_mk_atom((char*)atom), 1375 | ERL_DRV_TUPLE, 2}; 1376 | driver_send_term(state->port, state->port_owner, response, 1377 | sizeof(response) / sizeof(response[0])); 1378 | } 1379 | 1380 | static void send_error_str(PortState* state, const char* str) 1381 | { 1382 | ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("innostore_error"), 1383 | ERL_DRV_STRING, (ErlDrvTermData)str, (ErlDrvUInt)strlen(str), 1384 | ERL_DRV_TUPLE, 2}; 1385 | driver_send_term(state->port, state->port_owner, response, 1386 | sizeof(response) / sizeof(response[0])); 1387 | } 1388 | 1389 | -------------------------------------------------------------------------------- /c_src/innostore_drv.h: -------------------------------------------------------------------------------- 1 | // ------------------------------------------------------------------- 2 | // 3 | // innostore: Simple Erlang API to Embedded Inno DB 4 | // 5 | // Copyright (c) 2010 Basho Technologies, Inc. All Rights Reserved. 6 | // 7 | // innostore is free software: you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by 9 | // the Free Software Foundation, either version 2 of the License, or 10 | // (at your option) any later version. 11 | // 12 | // innostore is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with innostore. If not, see . 19 | // 20 | // ------------------------------------------------------------------- 21 | #ifndef INNOSTORE_DRV_H 22 | #define INNOSTORE_DRV_H 23 | 24 | #include "erl_driver.h" 25 | #include 26 | 27 | /** 28 | * Commands 29 | */ 30 | 31 | #define CMD_SET_CFG 1 32 | #define CMD_START (1 << 1) 33 | #define CMD_INIT_TABLE (1 << 2) 34 | #define CMD_IS_STARTED (1 << 3) 35 | #define CMD_GET (1 << 4) 36 | #define CMD_PUT (1 << 5) 37 | #define CMD_DELETE (1 << 6) 38 | #define CMD_LIST_TABLES (1 << 7) 39 | #define CMD_CURSOR_OPEN (1 << 8) 40 | #define CMD_CURSOR_MOVE (1 << 9) 41 | #define CMD_CURSOR_CLOSE (1 << 10) 42 | #define CMD_DROP_TABLE (1 << 11) 43 | #define CMD_STATUS (1 << 12) 44 | 45 | #define CMD_CURSOR_OPS CMD_CURSOR_MOVE | CMD_CURSOR_CLOSE 46 | 47 | /** 48 | * States 49 | */ 50 | #define STATE_READY 0 51 | #define STATE_CURSOR 1 52 | 53 | /** 54 | * Cursor movements/content requests 55 | */ 56 | #define CURSOR_FIRST 0 57 | #define CURSOR_NEXT 1 58 | #define CURSOR_PREV 2 59 | #define CURSOR_LAST 4 60 | 61 | #define CONTENT_KEY_ONLY 0 62 | #define CONTENT_KEY_VALUE 1 63 | 64 | #define FORMAT_REDUNDANT 1 65 | #define FORMAT_COMPACT 2 66 | #define FORMAT_DYNAMIC 3 67 | #define FORMAT_COMPRESSED 4 68 | 69 | /** 70 | * Operation function ptr 71 | */ 72 | typedef void (*op_fn_t)(void* arg); 73 | 74 | 75 | /** 76 | * Port State 77 | */ 78 | typedef struct 79 | { 80 | ErlDrvPort port; 81 | 82 | ErlDrvTermData port_owner; /* Pid of the port owner */ 83 | 84 | ErlDrvTid worker; /* Worker thread associated with this port */ 85 | 86 | ErlDrvMutex* worker_lock; 87 | 88 | ErlDrvCond* worker_cv; 89 | 90 | int port_state; 91 | 92 | int shutdown_flag; 93 | 94 | op_fn_t op; 95 | 96 | char* work_buffer; 97 | 98 | unsigned int work_buffer_sz; 99 | 100 | ib_trx_t txn; 101 | 102 | ib_crsr_t cursor; 103 | 104 | } PortState; 105 | 106 | 107 | /** 108 | * Helpful macros (from github/toland/bdberl) 109 | */ 110 | #define UNPACK_BYTE(_buf, _off) (_buf[_off]) 111 | #define UNPACK_INT(_buf, _off, _int_ptr) (memcpy((void*)_int_ptr, (_buf+_off), sizeof(*_int_ptr))) 112 | #define UNPACK_STRING(_buf, _off) (char*)(_buf+(_off)) 113 | #define UNPACK_BLOB(_buf, _off) (void*)(_buf+(_off)) 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /c_src/patches/disable_verbose_warnings.patch: -------------------------------------------------------------------------------- 1 | diff -ur embedded_innodb-1.0.6.6750.orig/include/api0api.h embedded_innodb-1.0.6.6750/include/api0api.h 2 | --- embedded_innodb-1.0.6.6750.orig/include/api0api.h 2010-02-22 05:57:13.000000000 -0700 3 | +++ embedded_innodb-1.0.6.6750/include/api0api.h 2010-03-01 10:14:09.000000000 -0700 4 | @@ -29,11 +29,7 @@ 5 | #define strcasecmp _stricmp 6 | #endif 7 | 8 | -#if defined(__GNUC__) && (__GNUC__ > 2) && ! defined(__INTEL_COMPILER) 9 | -#define UNIV_NO_IGNORE __attribute__ ((warn_unused_result)) 10 | -#else 11 | #define UNIV_NO_IGNORE 12 | -#endif /* __GNUC__ && __GNUC__ > 2 && !__INTEL_COMPILER */ 13 | 14 | /* See comment about ib_bool_t as to why the two macros are unsigned long. */ 15 | /** The boolean value of "true" used internally within InnoDB */ 16 | -------------------------------------------------------------------------------- /ebin/innostore.app: -------------------------------------------------------------------------------- 1 | {application, innostore, 2 | [{description, "Simple Erlang API to Embedded Inno DB"}, 3 | {vsn, "1.0.4"}, 4 | {modules, [ 5 | innostore, 6 | innostore_riak, 7 | riak_kv_innostore_backend 8 | ]}, 9 | {applications, [ 10 | kernel, 11 | stdlib, 12 | sasl 13 | ]}, 14 | {registered, []}, 15 | {env, [ 16 | %% Use current working directory for both log and data files by default 17 | {log_group_home_dir, "."}, 18 | {data_home_dir, "."}, 19 | 20 | {flush_log_at_trx_commit, 0}, % Flush pending log commits once per second 21 | {max_dirty_pages_pct, 75}, % Reduce frequency at which dirty pages are flushed 22 | {log_buffer_size, 8388608}, 23 | {format, compact} % default innodb format 24 | ]} 25 | ]}. 26 | 27 | -------------------------------------------------------------------------------- /package/Makefile: -------------------------------------------------------------------------------- 1 | ## Generate different PKGNAME's for each platform 2 | OS = $(shell uname -s) 3 | KERNEL = $(shell uname -r) 4 | ifeq ($(OS),Linux) 5 | PLATFORM = $(shell uname -m) 6 | PKGER = $(shell cat /etc/redhat-release 2> /dev/null) 7 | ifeq ($(PKGER),) 8 | # Debian 9 | PKGNAME = $(RELEASE_NAME)-$(RELEASE)-deb-$(PLATFORM).tar.gz 10 | else 11 | # Redhat/Fedora 12 | DISTRO = $(shell head -1 /etc/redhat-release | awk \ 13 | '{if ($$0 ~ /Fedora/) {print "fc"} else if ($$0 ~ /CentOS release 5/) {print "el5"} else if ($$0 ~ /CentOS release 6/) {print "el6"}}' ) 14 | PKGNAME = $(RELEASE_NAME)-$(RELEASE)-$(DISTRO)-$(PLATFORM).tar.gz 15 | endif 16 | endif 17 | ifeq ($(OS),SunOS) 18 | # Solaris 19 | PKG = BASHO$(REPO) 20 | PLATFORM = $(shell uname -p) 21 | SOLARIS_VER ?= $(shell echo "$(KERNEL)" | sed -e 's/^5\.//') 22 | DISTRO = $(shell awk '{ if (NR==1) print $$1; };' /etc/release) 23 | PKGNAME = $(PKG)-$(INNOSTORE_TAG)-$(RELEASE)-$(DISTRO)$(SOLARIS_VER)-$(PLATFORM).tar.gz 24 | endif 25 | ifeq ($(OS),Darwin) 26 | PLATFORM = $(shell uname -m) 27 | PKGNAME = $(RELEASE_NAME)-$(RELEASE)-osx-$(PLATFORM).tar.gz 28 | endif 29 | 30 | BUILDDIR = builddir 31 | PKGDIR = packages 32 | 33 | $(RELEASE_NAME).tar.gz: ../$(RELEASE_NAME).tar.gz 34 | ln -s $< $@ 35 | 36 | pkgclean: 37 | rm -rf $(RELEASE_NAME).tar.gz ${BUILDDIR} ${PKGDIR} 38 | 39 | pkgcheck: 40 | $(if $(INNOSTORE_TAG),,$(error "You can't generate a release tarball from a non-tagged revision. Run 'git checkout ', then 'make dist'")) 41 | $(if $(RELEASE),,$(error "You must provide a package release number via RELEASE= on the command line")) 42 | @echo "Packaging \"$(INNOSTORE_TAG)\"" 43 | 44 | build: buildrel 45 | @echo "Building package $(PKGNAME)" 46 | mkdir -p ${PKGDIR} 47 | cd $(BUILDDIR)/$(RELEASE_NAME) && \ 48 | mkdir -p $(RELEASE_NAME) && \ 49 | cp -R ebin $(RELEASE_NAME) && \ 50 | cp -R priv $(RELEASE_NAME) && \ 51 | cp -R src $(RELEASE_NAME) && \ 52 | tar -czf ../../${PKGDIR}/$(PKGNAME) $(RELEASE_NAME) 53 | 54 | buildrel: $(BUILDDIR)/$(RELEASE_NAME) 55 | cd $^ && $(MAKE) 56 | 57 | $(BUILDDIR)/$(RELEASE_NAME): $(BUILDDIR) $(RELEASE_NAME).tar.gz 58 | tar xz -C $(BUILDDIR) -f $(RELEASE_NAME).tar.gz 59 | 60 | $(BUILDDIR): 61 | mkdir -p $@ 62 | 63 | package: pkgcheck build 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /priv/innodump: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env escript 2 | %% -*- erlang -*- 3 | %% 4 | %% Innostore crash recovery - data dumper 5 | %% 6 | %% Iterates over all the keystores, dumping each one to a disklog names .log 7 | %% in the output directory. The disklog contains {K,V} tuples. The data can be 8 | %% reloaded using innoload. 9 | %% 10 | %% Increase verbosity with -v, shut it up with -q, default level is 1 11 | %% 0 - critical messages only 12 | %% 1 - basic progress messages - each keystore as it was dumped 13 | %% 2 - provides counts on each keystore as dumped 14 | %% 3 - dumps the key names as it goes 15 | %% 4 - dumps keys and values 16 | %% 17 | -record(state, {app_config, 18 | force_recover, 19 | output = "/tmp/innorecover", 20 | ignore = [], 21 | keep_going = false, 22 | verbosity = 1}). 23 | 24 | help() -> 25 | io:format("innodump - crash recovery for innostore data\n"), 26 | io:format("\n"), 27 | io:format("innodump [-v] [-q] [-config app.config] [-force_recovery 1-6] " 28 | "[-ignore keystore] [-output dir] [-keep-going]\n"). 29 | help(N) -> 30 | help(), 31 | halt(N). 32 | 33 | main(Args) -> 34 | State = parse_config(#state{}, Args), 35 | try 36 | find_ebin(), 37 | configure_innostore(State), 38 | case dump_keystores(State) of 39 | {ok, 0} -> 40 | ok; 41 | {ok, _N} -> 42 | safe_exit(State, read_error); 43 | {error, Reason} -> 44 | msg(State, 0, "InnoDB Error: ~p\n", [Reason]), 45 | safe_exit(State, Reason) 46 | end 47 | after 48 | shutdown_inno(State) 49 | end, 50 | msg(State, 2, "Clean exit\n"). 51 | 52 | %% 53 | %% Start inno and iterate over the keystores, dumping them to disklog files. 54 | %% 55 | dump_keystores(State) -> 56 | case innostore:connect() of 57 | {error, Reason} -> 58 | msg(State, 0, "Unable to start InnoDB"), 59 | {error, Reason}; 60 | 61 | {ok, P} -> 62 | Keystores = innostore:list_keystores(P), 63 | Rc = dump_each(State, P, Keystores, 0, 0), 64 | innostore:disconnect(P), 65 | Rc 66 | end. 67 | 68 | 69 | dump_each(State, _, [], Total, Errors) -> 70 | msg(State, 1, "Completed: ~p k/v pairs with ~p errors.\n", [Total, Errors]), 71 | {ok, Errors}; 72 | dump_each(State, Port, [Keystore | Rest], Total, Errors) -> 73 | case lists:member(Keystore, State#state.ignore) of 74 | true -> 75 | msg(State, 1, "Ignoring ~p\n", [Keystore]), 76 | dump_each(State, Port, Rest, Total, Errors); 77 | 78 | false -> 79 | TargetFile = filename:join(State#state.output, Keystore ++ ".log"), 80 | case dump_keystore(State, Port, TargetFile, Keystore) of 81 | {error, Reason} -> 82 | %% Decide what to do 83 | msg(State, 0, "Dumping ~p had errors - ~p", [Keystore, Reason]), 84 | case State#state.keep_going of 85 | true -> 86 | msg(State, 1, " -- trying to keep going\n"), 87 | dump_each(State, Port, Rest, Total, Errors + 1); 88 | 89 | _ -> 90 | msg(State, 0, "\nStopping - try increasing -force_recovery " 91 | "or setting -keep-going\n"), 92 | safe_exit(State, read_errors) 93 | end; 94 | {_, Count, _} -> 95 | msg(State, 2, "Dumped ~p -> ~p - ~p k/v pairs.\n", 96 | [Keystore, TargetFile, Count]), 97 | dump_each(State, Port, Rest, Total + Count, Errors) 98 | end 99 | end. 100 | 101 | 102 | dump_keystore(State, Port, TargetFile, Keystore) -> 103 | case innostore:open_keystore(Keystore, Port) of 104 | {ok, Ks} -> 105 | ok = filelib:ensure_dir(TargetFile), 106 | {ok, TargetLog} = disk_log:open([{name, TargetFile}, 107 | {file, TargetFile}, 108 | {repair, truncate}]), 109 | try 110 | msg(State, 1, "Dumping ~p -> ~p\n", [Keystore, TargetFile]), 111 | dump_kvs(State, Ks, TargetLog) 112 | after 113 | disk_log:close(TargetLog) 114 | end; 115 | X -> 116 | X 117 | end. 118 | 119 | 120 | dump_kvs(State, Ks, TargetLog) -> 121 | F = fun(K, V, {_State, Count, Log}) -> 122 | ok = disk_log:alog(Log, {K, V}), 123 | Count1 = Count + 1, 124 | if 125 | State#state.verbosity > 3 -> 126 | io:format("~p: Key: ~p\nValue: ~p\n", [Count1, K, V]); 127 | State#state.verbosity =:= 3 -> 128 | io:format("~p: Key: ~p\n", [Count1, K]); 129 | State#state.verbosity =:= 1, Count1 rem 10000 =:= 0 -> 130 | io:format("Processed: ~p\n", [Count1]); 131 | true -> 132 | ok 133 | end, 134 | {State, Count1, Log} 135 | end, 136 | innostore:fold(F, {State, 0, TargetLog}, Ks). 137 | 138 | 139 | %% 140 | %% Work out where ebin is and add to the code path 141 | %% 142 | find_ebin() -> 143 | case code:which(innostore) of 144 | non_existing -> 145 | case find_inno_root(escript:script_name()) of 146 | {ok, InnostoreRoot} -> 147 | EbinDir = filename:join(InnostoreRoot ++ ["ebin"]), 148 | true = code:add_patha(EbinDir); 149 | _ -> 150 | ok 151 | end, 152 | case code:ensure_loaded(innostore) of 153 | {module, innostore} -> 154 | ok; 155 | X -> 156 | io:format("Could not load innostore - ~p\n", [X]), 157 | halt(1) 158 | end; 159 | _Exists -> 160 | ok 161 | end. 162 | 163 | %% Try and find the root directory for innostore 164 | find_inno_root(Scriptname) -> 165 | case lists:reverse(filename:split(Scriptname)) of 166 | ["innodump", "priv" | Rest] -> 167 | {ok, lists:reverse(Rest)}; 168 | S -> 169 | io:format("~p\n", [S]), 170 | not_found 171 | end. 172 | 173 | 174 | %% 175 | %% Load the innostore defaults, override them with the application config 176 | %% and set force_recovery if required 177 | %% 178 | configure_innostore(State) -> 179 | application:load(innostore), 180 | import_config(State#state.app_config), 181 | case State#state.force_recover of 182 | undefined -> 183 | ok; 184 | Level -> 185 | msg(State, 1, "Setting force recovery to ~p\n\n", [Level]), 186 | application:set_env(innostore, force_recovery, Level) 187 | end, 188 | msg(State, 2, "Innostore running with configuration\n~p\n", 189 | [application:get_all_env(innostore)]). 190 | 191 | %% 192 | %% Import the innostore section from an app.config 193 | %% 194 | import_config(undefined) -> 195 | ok; 196 | import_config(AppState) -> 197 | case file:consult(AppState) of 198 | {ok, State} -> 199 | find_innostore(State); 200 | {error, Reason} -> 201 | msg(0, "Could not ready \"~s\" - ~p", [AppState, Reason]) 202 | end. 203 | 204 | %% Look for {innostore, [AppSpecKeys]} somewhere in a list 205 | find_innostore([]) -> 206 | not_found; 207 | find_innostore([{innostore, AppSpecKeys} | _Rest]) when is_list(AppSpecKeys) -> 208 | set_config(AppSpecKeys); 209 | find_innostore([This | Rest]) when is_list(This) -> 210 | case find_innostore(This) of 211 | not_found -> 212 | find_innostore(Rest); 213 | X -> 214 | X 215 | end; 216 | find_innostore([_This | Rest]) -> % ignore any non-innostore atoms/tuples 217 | find_innostore(Rest). 218 | 219 | %% Set the application environment 220 | set_config([]) -> 221 | ok; 222 | set_config([{Par, Val} | Rest]) -> 223 | application:set_env(innostore, Par, Val), 224 | set_config(Rest). 225 | 226 | 227 | %% Make sure innostore is unloaded properly so it will call the C ib_shutdown() function. 228 | shutdown_inno(State) -> 229 | case erl_ddll:try_unload(innostore_drv, [{monitor, pending_driver}, kill_ports]) of 230 | {ok, pending_driver, Ref} -> 231 | msg(State, 2, "Unloading Innostore\n"), 232 | receive 233 | {'DOWN', Ref, driver, innostore_drv, unloaded} -> 234 | ok; 235 | X -> 236 | msg(State, 0, "Unexpected message: ~p\n", [X]) 237 | end; 238 | {ok, unloaded} -> 239 | ok; 240 | {error, not_loaded} -> 241 | ok 242 | end, 243 | msg(State, 2, "Innostore unloaded\n"). 244 | 245 | 246 | %% 247 | %% Safe exit function - make sure innostore is unloaded before calling halt() 248 | %% 249 | safe_exit(State, Why) -> 250 | shutdown_inno(State), 251 | 252 | %% Make a nice exit code 253 | Code = case Why of 254 | startup_error -> 255 | 1; 256 | read_error -> 257 | 2; 258 | config -> 259 | 3; 260 | _ -> 261 | 10 262 | end, 263 | halt(Code). 264 | 265 | 266 | %% 267 | %% Output a logging message to the user if the verbosity is set high enough 268 | %% 269 | msg(State, Level, Msg) when State#state.verbosity >= Level -> 270 | io:format(Msg); 271 | msg(_State, _Level, _Msg) -> 272 | ok. 273 | 274 | msg(State, Level, Msg, Data) when State#state.verbosity >= Level -> 275 | io:format(Msg, Data); 276 | msg(_State, _Level, _Msg, _Data) -> 277 | ok. 278 | 279 | %% 280 | %% Argument parsing 281 | %% 282 | parse_config(State, []) -> 283 | State; 284 | parse_config(_State, ["-help" | _Rest]) -> 285 | help(0); 286 | parse_config(State, ["-config" | Rest]) -> 287 | parse_config(State#state{app_config = expect_str(Rest) }, tl(Rest)); 288 | parse_config(State, ["-force_recovery" | Rest]) -> 289 | parse_config(State#state{force_recover = expect_level(Rest)}, tl(Rest)); 290 | parse_config(State, ["-output" | Rest]) -> 291 | parse_config(State#state{output = expect_dir(Rest)}, tl(Rest)); 292 | parse_config(State, ["-ignore" | Rest]) -> 293 | parse_config(State#state{ignore = [expect_dir(Rest) | State#state.ignore]}, tl(Rest)); 294 | parse_config(State, ["-keep-going" | Rest]) -> 295 | parse_config(State#state{keep_going = true}, Rest); 296 | parse_config(State, ["-v" | Rest]) -> 297 | parse_config(State#state{verbosity = State#state.verbosity + 1}, Rest); 298 | parse_config(State, ["-q" | Rest]) -> 299 | parse_config(State#state{verbosity = 0}, Rest); 300 | parse_config(State, [[$-, $- | Arg] | Rest]) -> % Also handle options as --option 301 | parse_config(State, [[$-| Arg] | Rest]); 302 | parse_config(_State, ["-help" | _Rest]) -> 303 | help(0); 304 | parse_config(_State, [Other | _Rest]) -> 305 | io:format("Unexpected option \"~s\"\n", [Other]), 306 | help(1). 307 | 308 | %% Expect a string, help and exit otherwise 309 | expect_str([]) -> 310 | io:format("Expecting string\n"), 311 | help(2); 312 | expect_str([Str|_]) -> 313 | Str. 314 | 315 | %% Expect a recovery level between 1 and 6, help and exit otherwise 316 | expect_level([]) -> 317 | io:format("Expecting recovery level\n"), 318 | help(2); 319 | expect_level([LevelStr|_]) -> 320 | Level = list_to_integer(LevelStr), 321 | case Level < 1 orelse Level > 6 of 322 | true -> 323 | io:format("Recovery level must be between 1 and 6\n"), 324 | help(2); 325 | false -> 326 | Level 327 | end. 328 | 329 | %% Expect a recovery level between 1 and 6, help and exit otherwise 330 | expect_dir([Dir|_]) -> 331 | case filelib:ensure_dir([Dir, "foo"]) of 332 | {error, Reason} -> 333 | io:format("Problems checking/creating directory ~s - ~p\n", [Dir, Reason]), 334 | help(2); 335 | ok -> 336 | Dir 337 | end. 338 | 339 | -------------------------------------------------------------------------------- /priv/innoload: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env escript 2 | %% -*- erlang -*- 3 | %% 4 | %% Innostore crash recovery - data loader 5 | %% 6 | %% Iterates over all the .log files created by innodump loading into . 7 | %% The disklog contains {K,V} tuples. 8 | %% 9 | %% Increase verbosity with -v, shut it up with -q, default level is 1 10 | %% 0 - critical messages only 11 | %% 1 - basic progress messages/counts - each keystore as it is loaded 12 | %% 2 - some extra detail 13 | 14 | -record(state, { 15 | %% Configuration 16 | app_config, 17 | input, 18 | keep_going = false, 19 | verbosity = 1, 20 | 21 | %% Runtime 22 | count = 0, 23 | total = 0, 24 | errors = 0 25 | }). 26 | 27 | help() -> 28 | io:format("innoload - crash recovery for innostore data\n"), 29 | io:format("\n"), 30 | io:format("innoload [-v] [-q] [-config app.config] " 31 | "[-input dir] [-keep-going]\n"). 32 | help(N) -> 33 | help(), 34 | halt(N). 35 | 36 | main(Args) -> 37 | State = parse_config(#state{}, Args), 38 | try 39 | find_ebin(), 40 | configure_innostore(State), 41 | case load_keystores(State) of 42 | {ok, State1} when State1#state.errors =:= 0 -> 43 | ok; 44 | {ok, State1} -> 45 | safe_exit(State1, read_error); 46 | {{error, Reason}, State1} -> 47 | msg(State1, 0, "InnoDB Error: ~p\n", [Reason]), 48 | safe_exit(State, Reason) 49 | end 50 | after 51 | shutdown_inno(State) 52 | end, 53 | msg(State, 2, "Clean exit\n"). 54 | 55 | %% 56 | %% Start inno and iterate over the keystores, dumping them to disklog files. 57 | %% 58 | load_keystores(State) -> 59 | case innostore:connect() of 60 | {error, Reason} -> 61 | msg(State, 0, "Unable to start InnoDB\n"), 62 | {error, Reason}; 63 | 64 | {ok, P} -> 65 | Sources = find_log_files(State), 66 | Rc = load_each(State, P, Sources), 67 | innostore:disconnect(P), 68 | Rc 69 | end. 70 | 71 | 72 | load_each(State, _, []) -> 73 | msg(State, 1, "Completed: ~p k/v pairs with ~p errors.\n", [State#state.total, 74 | State#state.errors]), 75 | {ok, State}; 76 | load_each(State, Port, [LogFile | Rest]) -> 77 | case load_each_disklog(State, Port, LogFile) of 78 | {ok, State1} -> 79 | load_each(State1, Port, Rest); 80 | 81 | {{error, Reason}, State1} -> 82 | msg(State, 1, "Loading ~p had errors - ~p", [LogFile, Reason]), 83 | State1 = State#state{errors = State#state.errors + 1}, 84 | case State1#state.keep_going of 85 | true -> 86 | msg(State1, 1, " -- trying to keep going\n"), 87 | load_each(State1, Port, Rest); 88 | 89 | _ -> 90 | msg(State1, 1, "\nStopping - try increasing -force_recovery " 91 | "or setting -keep-going\n"), 92 | safe_exit(State1, load_errors) 93 | end 94 | end. 95 | 96 | load_each_disklog(State, Port, LogFile) -> 97 | Keystore = filename:basename(LogFile, ".log"), 98 | msg(State, 1, "Loading ~s -> ~s\n", [LogFile, Keystore]), 99 | 100 | case disk_log:open([{name, LogFile}, {file, LogFile}, {mode, read_only}]) of 101 | {ok, Log} -> 102 | try 103 | case innostore:open_keystore(Keystore, Port) of 104 | {ok, Ks} -> 105 | load_kvs(State, Log, start, Ks); 106 | X -> 107 | {X, State} 108 | end 109 | catch 110 | Class:Reason -> 111 | {{error, {Class, Reason}}, State} 112 | after 113 | disk_log:close(Log) 114 | end; 115 | X -> 116 | {X, State} 117 | end. 118 | 119 | 120 | load_kvs(State, Log, Cont, Ks) -> 121 | case disk_log:chunk(Log, Cont) of 122 | eof -> 123 | msg(State, 1, "... loaded ~p k/v pairs.\n", [State#state.count]), 124 | {ok, State#state{total = State#state.total + State#state.count, count = 0}}; 125 | {Cont2, Terms} -> 126 | {ok, State1} = insert_terms(State, Terms, Ks), 127 | load_kvs(State1, Log, Cont2, Ks) 128 | end. 129 | 130 | insert_terms(State, [], _Ks) -> 131 | {ok, State}; 132 | insert_terms(State, [{K, V} | Rest], Ks) -> 133 | ok = innostore:put(K, V, Ks), 134 | Count1 = State#state.count + 1, 135 | insert_terms(State#state{count = Count1}, Rest, Ks). 136 | 137 | 138 | 139 | %% 140 | %% Work out where ebin is and add to the code path 141 | %% 142 | find_ebin() -> 143 | case code:which(innostore) of 144 | non_existing -> 145 | case find_inno_root(escript:script_name()) of 146 | {ok, InnostoreRoot} -> 147 | EbinDir = filename:join(InnostoreRoot ++ ["ebin"]), 148 | true = code:add_patha(EbinDir); 149 | _ -> 150 | ok 151 | end, 152 | case code:ensure_loaded(innostore) of 153 | {module, innostore} -> 154 | ok; 155 | X -> 156 | io:format("Could not load innostore - ~p\n", [X]), 157 | halt(1) 158 | end; 159 | _Exists -> 160 | ok 161 | end. 162 | 163 | %% Try and find the root directory for innostore 164 | find_inno_root(Scriptname) -> 165 | case lists:reverse(filename:split(Scriptname)) of 166 | ["innodump", "priv" | Rest] -> 167 | {ok, lists:reverse(Rest)}; 168 | S -> 169 | not_found 170 | end. 171 | 172 | 173 | %% 174 | %% Load the innostore defaults, override them with the application state 175 | %% and set force_recovery if required 176 | %% 177 | configure_innostore(State) -> 178 | application:load(innostore), 179 | import_config(State#state.app_config), 180 | msg(State, 2, "Innostore running with configuration\n~p\n", 181 | [application:get_all_env(innostore)]). 182 | 183 | import_config(undefined) -> 184 | ok; 185 | import_config(AppConfig) -> 186 | case file:consult(AppConfig) of 187 | {ok, Config} -> 188 | find_innostore(Config); 189 | {error, Reason} -> 190 | msg(0, "Could not ready \"~s\" - ~p", [AppConfig, Reason]) 191 | end. 192 | 193 | %% Look for {innostore, [AppSpecKeys]} somewhere in a list 194 | find_innostore([]) -> 195 | not_found; 196 | find_innostore([{innostore, AppSpecKeys} | _Rest]) when is_list(AppSpecKeys) -> 197 | set_config(AppSpecKeys); 198 | find_innostore([This | Rest]) when is_list(This) -> 199 | case find_innostore(This) of 200 | not_found -> 201 | find_innostore(Rest); 202 | X -> 203 | X 204 | end; 205 | find_innostore([_This | Rest]) -> % ignore any non-innostore atoms/tuples 206 | find_innostore(Rest). 207 | 208 | %% Set the application environment 209 | set_config([]) -> 210 | ok; 211 | set_config([{Par, Val} | Rest]) -> 212 | application:set_env(innostore, Par, Val), 213 | set_config(Rest). 214 | 215 | %% 216 | %% Find all .log files in the input directory 217 | %% 218 | find_log_files(State) -> 219 | Spec = case State#state.input of 220 | undefined -> 221 | "*.log"; 222 | Dir -> 223 | filename:join([Dir, "*.log"]) 224 | end, 225 | filelib:wildcard(Spec). 226 | 227 | 228 | %% Make sure innostore is unloaded properly so it will call the C ib_shutdown() function. 229 | shutdown_inno(State) -> 230 | case erl_ddll:try_unload(innostore_drv, [{monitor, pending_driver}, kill_ports]) of 231 | {ok, pending_driver, Ref} -> 232 | msg(State, 2, "Unloading Innostore\n"), 233 | receive 234 | {'DOWN', Ref, driver, innostore_drv, unloaded} -> 235 | ok; 236 | X -> 237 | io:format("Unexpected message: ~p\n", [X]) 238 | end; 239 | {ok, unloaded} -> 240 | ok; 241 | {error, not_loaded} -> 242 | ok; 243 | X -> 244 | io:format("Unexpected return: ~p\n", [X]) 245 | end, 246 | msg(State, 2, "Innostore unloaded\n"). 247 | 248 | 249 | %% 250 | %% Safe exit function - make sure innostore is unloaded before calling halt() 251 | %% 252 | safe_exit(State, Why) -> 253 | shutdown_inno(State), 254 | 255 | %% Make a nice exit code 256 | Code = case Why of 257 | startup_error -> 258 | 1; 259 | read_error -> 260 | 2; 261 | state -> 262 | 3; 263 | _ -> 264 | 10 265 | end, 266 | halt(Code). 267 | 268 | 269 | %% 270 | %% Output a logging message to the user if the verbosity is set high enough 271 | %% 272 | 273 | msg(State, Level, Msg) when State#state.verbosity >= Level -> 274 | io:format(Msg); 275 | msg(_State, _Level, _Msg) -> 276 | ok. 277 | 278 | msg(State, Level, Msg, Data) when State#state.verbosity >= Level -> 279 | io:format(Msg, Data); 280 | msg(_State, _Level, _Msg, _Data) -> 281 | ok. 282 | 283 | %% 284 | %% Argument parsing 285 | %% 286 | 287 | parse_config(State, []) -> 288 | State; 289 | parse_config(_State, ["-help" | _Rest]) -> 290 | help(0); 291 | parse_config(State, ["-config" | Rest]) -> 292 | parse_config(State#state{app_config = expect_str(Rest) }, tl(Rest)); 293 | parse_config(State, ["-input" | Rest]) -> 294 | parse_config(State#state{input = expect_dir(Rest)}, tl(Rest)); 295 | parse_config(State, ["-keep-going" | Rest]) -> 296 | parse_config(State#state{keep_going = true}, tl(Rest)); 297 | parse_config(State, ["-v" | Rest]) -> 298 | parse_config(State#state{verbosity = State#state.verbosity + 1}, Rest); 299 | parse_config(State, ["-q" | Rest]) -> 300 | parse_config(State#state{verbosity = 0}, Rest); 301 | parse_config(State, [[$-, $- | Arg] | Rest]) -> % Also handle options as --option 302 | parse_config(State, [[$-| Arg] | Rest]); 303 | parse_config(_State, ["-help" | _Rest]) -> 304 | help(0); 305 | parse_config(_State, [Other | _Rest]) -> 306 | io:format("Unexpected option \"~s\"\n", [Other]), 307 | help(1). 308 | 309 | 310 | expect_str([]) -> 311 | io:format("Expecting string\n"), 312 | help(2); 313 | expect_str([Str|_]) -> 314 | Str. 315 | 316 | expect_dir([Dir|_]) -> 317 | case filelib:ensure_dir([Dir, "foo"]) of 318 | {error, Reason} -> 319 | io:format("Problems checking/creating directory ~s - ~p\n", [Dir, Reason]), 320 | help(2); 321 | ok -> 322 | Dir 323 | end. 324 | 325 | -------------------------------------------------------------------------------- /priv/riak-innostore: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 4 | 5 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*/*/*} 6 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 7 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 8 | RUNNER_USER= 9 | 10 | # Make sure this script is running as the appropriate user 11 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then 12 | exec sudo -u $RUNNER_USER -i $0 $@ 13 | fi 14 | 15 | # Make sure CWD is set to runner base dir 16 | cd $RUNNER_BASE_DIR 17 | if [ ! -d $RUNNER_ETC_DIR ] 18 | then 19 | echo "cannot find riak." 20 | exit 1 21 | fi 22 | 23 | # Extract the target node name from node.args 24 | NAME_ARG=`grep '\-[s]*name' $RUNNER_ETC_DIR/vm.args` 25 | if [ -z "$NAME_ARG" ]; then 26 | echo "vm.args needs to have either -name or -sname parameter." 27 | exit 1 28 | fi 29 | 30 | # Learn how to specify node name for connection from remote nodes 31 | echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 32 | if [ "X$?" = "X0" ]; then 33 | NAME_PARAM="-sname" 34 | NAME_HOST="" 35 | else 36 | NAME_PARAM="-name" 37 | echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 38 | if [ "X$?" = "X0" ]; then 39 | NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*\(@.*\)$/\1/'` 40 | else 41 | NAME_HOST="" 42 | fi 43 | fi 44 | 45 | # Extract the target cookie 46 | COOKIE_ARG=`grep '\-setcookie' $RUNNER_ETC_DIR/vm.args` 47 | if [ -z "$COOKIE_ARG" ]; then 48 | echo "vm.args needs to have a -setcookie parameter." 49 | exit 1 50 | fi 51 | 52 | # Identify the script name 53 | SCRIPT=`basename $0` 54 | 55 | # Parse out release and erts info 56 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 57 | ERTS_VSN=${START_ERL% *} 58 | APP_VSN=${START_ERL#* } 59 | 60 | # Add ERTS bin dir to our path 61 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 62 | 63 | # Setup command to control the node 64 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 65 | 66 | # Check the first argument for instructions 67 | case "$1" in 68 | status) 69 | # Make sure the local node IS running 70 | RES=`$NODETOOL ping` 71 | if [ "$RES" != "pong" ]; then 72 | echo "Node is not running!" 73 | exit 1 74 | fi 75 | shift 76 | 77 | $NODETOOL rpc innostore_riak status $@ 78 | ;; 79 | *) 80 | echo "Usage: $SCRIPT { status }" 81 | exit 1 82 | ;; 83 | esac 84 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/innostore/fd487cd0c2ce220b25f007450b1a29e81bb05d17/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {so_name, "innostore_drv.so"}. 2 | {erl_opts, [debug_info, warnings_as_errors]}. 3 | {deps, []}. 4 | 5 | {port_envs, [ 6 | {"DRV_CFLAGS", "$DRV_CFLAGS -Werror -I c_src/innodb/include"}, 7 | {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/innodb/lib/libinnodb.a"}, 8 | 9 | %% Solaris specific flags 10 | {"solaris.*-64$", "CFLAGS", "-D_REENTRANT -m64"}, 11 | {"solaris.*-64$", "LDFLAGS", "-m64"}, 12 | 13 | %% OS X Leopard flags for 64-bit 14 | {"darwin9.*-64$", "CFLAGS", "-m64"}, 15 | {"darwin9.*-64$", "LDFLAGS", "-arch x86_64"}, 16 | 17 | %% OS X Snow Leopard flags for 32-bit 18 | {"darwin10.*-32$", "CFLAGS", "-m32"}, 19 | {"darwin10.*-32$", "LDFLAGS", "-arch i386"} 20 | ]}. 21 | 22 | {pre_hooks, [{compile, "c_src/build_deps.sh"}]}. 23 | 24 | {post_hooks, [{clean, "c_src/build_deps.sh clean"}]}. 25 | 26 | {cover_enabled, true}. 27 | -------------------------------------------------------------------------------- /src/innostore.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% innostore: Simple Erlang API to Embedded Inno DB 4 | %% 5 | %% Copyright (c) 2009 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% innostore is free software: you can redistribute it and/or modify 8 | %% it under the terms of the GNU General Public License as published by 9 | %% the Free Software Foundation, either version 2 of the License, or 10 | %% (at your option) any later version. 11 | %% 12 | %% innostore is distributed in the hope that it will be useful, 13 | %% but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | %% GNU General Public License for more details. 16 | %% 17 | %% You should have received a copy of the GNU General Public License 18 | %% along with innostore. If not, see . 19 | %% 20 | %% ------------------------------------------------------------------- 21 | -module(innostore). 22 | 23 | -author('Dave Smith '). 24 | 25 | %% Public API 26 | -export([connect/0, 27 | disconnect/1, 28 | open_keystore/2, open_keystore/3, 29 | is_keystore_empty/2, 30 | list_keystores/1, 31 | drop_keystore/2, 32 | get/2, 33 | put/3, 34 | delete/2, 35 | fold_keys/3, 36 | fold/3, 37 | status/1, status/2]). 38 | 39 | 40 | -ifdef(TEST). 41 | -include_lib("eunit/include/eunit.hrl"). 42 | -endif. 43 | 44 | -define(CMD_SET_CFG, 1). 45 | -define(CMD_START, 1 bsl 1). 46 | -define(CMD_INIT_TABLE, 1 bsl 2). 47 | -define(CMD_IS_STARTED, 1 bsl 3). 48 | -define(CMD_GET, 1 bsl 4). 49 | -define(CMD_PUT, 1 bsl 5). 50 | -define(CMD_DELETE, 1 bsl 6). 51 | -define(CMD_LIST_TABLES, 1 bsl 7). 52 | -define(CMD_CURSOR_OPEN, 1 bsl 8). 53 | -define(CMD_CURSOR_MOVE, 1 bsl 9). 54 | -define(CMD_CURSOR_CLOSE, 1 bsl 10). 55 | -define(CMD_DROP_TABLE, 1 bsl 11). 56 | -define(CMD_STATUS, 1 bsl 12). 57 | 58 | -define(CURSOR_FIRST, 0). 59 | -define(CURSOR_NEXT, 1). 60 | -define(CURSOR_PREV, 2). 61 | -define(CURSOR_LAST, 3). 62 | 63 | -define(CONTENT_KEY_ONLY, 0). 64 | -define(CONTENT_KEY_VALUE, 1). 65 | 66 | -define(COMPRESSION_NONE, 0). 67 | 68 | -define(FORMAT_REDUNDANT, 1). 69 | -define(FORMAT_COMPACT, 2). 70 | -define(FORMAT_DYNAMIC, 3). 71 | -define(FORMAT_COMPRESSED, 4). 72 | 73 | -record(store, { port, 74 | table_id, 75 | compression = ?COMPRESSION_NONE }). 76 | 77 | 78 | %% =================================================================== 79 | %% Public API 80 | %% =================================================================== 81 | 82 | connect() -> 83 | case erl_ddll:load_driver(priv_dir(), innostore_drv) of 84 | Res when Res == ok; Res == {error, permanent} -> 85 | Port = open_port({spawn, innostore_drv}, [binary]), 86 | case is_started(Port) of 87 | true -> 88 | {ok, Port}; 89 | false -> 90 | ensure_app_loaded(), 91 | set_config(application:get_all_env(innostore), Port), 92 | try_and_start(Port) 93 | end; 94 | 95 | {error, LoadError} -> 96 | Str = erl_ddll:format_error(LoadError), 97 | error_logger:error_msg("Error loading driver ~s: ~p\n", [?MODULE, Str]), 98 | throw({error, {LoadError, Str}}) 99 | end. 100 | 101 | disconnect(Port) -> 102 | port_close(Port), 103 | ok. 104 | 105 | 106 | open_keystore(Name, Port) -> 107 | open_keystore(Name, config_format(), Port). 108 | 109 | open_keystore(Name, Format, Port) when is_atom(Name) -> 110 | open_keystore(atom_to_binary(Name, utf8), Format, Port); 111 | open_keystore(Name, Format, Port) when is_list(Name) -> 112 | open_keystore(list_to_binary(Name), Format, Port); 113 | open_keystore(Name, Format, Port) when is_binary(Name) -> 114 | TableName = <<"innokeystore/", Name/binary>>, 115 | 116 | erlang:port_control(Port, ?CMD_INIT_TABLE, 117 | <<(format_encode(Format)):8, TableName/binary, 0:8>>), 118 | receive 119 | {innostore_ok, <<_:64/unsigned-native>> = TableId} -> 120 | {ok, #store { port = Port, 121 | table_id = TableId }}; 122 | 123 | {innostore_error, Reason}-> 124 | {error, Reason} 125 | end. 126 | 127 | is_keystore_empty(Name, Port) -> 128 | case open_keystore(Name, Port) of 129 | {ok, Store} -> 130 | ok = cursor_open(Store), 131 | case cursor_move(?CURSOR_FIRST, ?CONTENT_KEY_ONLY, Store) of 132 | {ok, eof} -> 133 | Result = true; 134 | _ -> 135 | Result = false 136 | end, 137 | ok = cursor_close(Store), 138 | Result; 139 | {error, Reason} -> 140 | {error, Reason} 141 | end. 142 | 143 | list_keystores(Port) -> 144 | erlang:port_control(Port, ?CMD_LIST_TABLES, <<>>), 145 | list_keystores_loop([]). 146 | 147 | 148 | drop_keystore(Name, Port) when is_atom(Name) -> 149 | drop_keystore(atom_to_binary(Name, utf8), Port); 150 | drop_keystore(Name, Port) when is_list(Name) -> 151 | drop_keystore(list_to_binary(Name), Port); 152 | drop_keystore(Name, Port) -> 153 | TableName = <<"innokeystore/", Name/binary>>, 154 | erlang:port_control(Port, ?CMD_DROP_TABLE, <>), 155 | receive 156 | innostore_ok -> 157 | ok; 158 | {innostore_error, Reason} -> 159 | {error, Reason} 160 | end. 161 | 162 | 163 | get(Key, _Store) when size(Key) > 255 -> 164 | {error, key_exceeds_255_bytes}; 165 | get(Key, Store) -> 166 | Args = <<(Store#store.table_id)/binary, (size(Key)):8, Key/binary>>, 167 | erlang:port_control(Store#store.port, ?CMD_GET, Args), 168 | receive 169 | {innostore_ok, not_found} -> 170 | {ok, not_found}; 171 | {innostore_ok, Value} -> 172 | {ok, Value}; 173 | {innostore_error, Reason} -> 174 | {error, Reason} 175 | end. 176 | 177 | 178 | put(Key, _Value, _Store) when size(Key) > 255 -> 179 | {error, key_exceeds_255_bytes}; 180 | put(Key, Value, Store) -> 181 | Args = <<(Store#store.table_id)/binary, (Store#store.compression):8, 182 | (size(Key)):8, Key/binary, 183 | (size(Value)):32/native, Value/binary>>, 184 | erlang:port_control(Store#store.port, ?CMD_PUT, Args), 185 | receive 186 | innostore_ok -> 187 | ok; 188 | {innostore_error, Reason} -> 189 | {error, Reason} 190 | end. 191 | 192 | delete(Key, _Store) when size(Key) > 255 -> 193 | {error, key_exceeds_255_bytes}; 194 | delete(Key, Store) -> 195 | Args = <<(Store#store.table_id)/binary, (size(Key)):8, Key/binary>>, 196 | erlang:port_control(Store#store.port, ?CMD_DELETE, Args), 197 | receive 198 | innostore_ok -> 199 | ok; 200 | {innostore_error, Reason} -> 201 | {error, Reason} 202 | end. 203 | 204 | fold_keys(Fun, Acc0, Store) -> 205 | fold(Fun, Acc0, ?CONTENT_KEY_ONLY, Store). 206 | 207 | fold(Fun, Acc0, Store) -> 208 | fold(Fun, Acc0, ?CONTENT_KEY_VALUE, Store). 209 | 210 | 211 | status(Name, Port) when is_atom(Name) -> 212 | Var = atom_to_binary(Name, latin1), 213 | erlang:port_control(Port, ?CMD_STATUS, <>), 214 | receive 215 | {innostore_ok, <>} -> 216 | Value; 217 | {innostore_error, Reason} -> 218 | {error, Reason} 219 | end. 220 | 221 | status(Port) -> 222 | [{S, status(S, Port)} || S <- status_names()]. 223 | 224 | 225 | %% =================================================================== 226 | %% Internal functions 227 | %% =================================================================== 228 | 229 | priv_dir() -> 230 | case code:priv_dir(?MODULE) of 231 | Name when is_list(Name) -> 232 | Name; 233 | {error, bad_name} -> 234 | {ok, Cwd} = file:get_cwd(), 235 | filename:absname(filename:join(Cwd, "../priv")) 236 | end. 237 | 238 | is_started(Port) -> 239 | erlang:port_control(Port, ?CMD_IS_STARTED, <<>>) == <<1>>. 240 | 241 | ensure_app_loaded() -> 242 | case lists:keymember(?MODULE, 1, application:loaded_applications()) of 243 | true -> 244 | ok; 245 | false -> 246 | case application:load(?MODULE) of 247 | ok -> 248 | ok; 249 | {error, _Reason} -> 250 | error_logger:info_msg("Using default Innostore configuration.\n") 251 | end 252 | end. 253 | 254 | set_config([], _Port) -> 255 | ok; 256 | set_config([{included_applications, _} | Rest], Port) -> 257 | set_config(Rest, Port); 258 | set_config([{format, _} | Rest], Port) -> 259 | set_config(Rest, Port); 260 | set_config([{Key, Value} | Rest], Port) when is_atom(Key) -> 261 | case lists:keysearch(Key, 1, config_types()) of 262 | {value, {Key, Type}} -> 263 | KBin = atom_to_binary(Key, utf8), 264 | VBin = config_encode(Type, Key, Value), 265 | erlang:port_control(Port, ?CMD_SET_CFG, <>), 266 | receive 267 | innostore_ok -> 268 | case on_set_config(Key, Value) of 269 | ok -> 270 | ok; 271 | {error, Reason} -> 272 | error_logger:error_msg("Failed to post-process value for ~p = ~p: ~p\n", 273 | [Key, Value, Reason]) 274 | end; 275 | {innostore_error, starting} -> % stop setting config - we are starting 276 | ok; 277 | {innostore_error, Reason} -> 278 | error_logger:error_msg("Failed to set value for ~p = ~p: ~p\n", [Key, Value, Reason]) 279 | end; 280 | false -> 281 | error_logger:error_msg("Skipping config setting ~p; unknown option.\n", [Key]) 282 | end, 283 | set_config(Rest, Port); 284 | set_config([Other | Rest], Port) -> 285 | error_logger:info_msg("Skipping config setting ~p for innostore; not {atom, list} pair.\n", 286 | [Other]), 287 | set_config(Rest, Port). 288 | 289 | %% 290 | %% Post process values set in the config 291 | %% 292 | on_set_config(Key, Dir) when Key == log_group_home_dir; Key == data_home_dir -> 293 | filelib:ensure_dir(filename:join(Dir, "foo")); 294 | on_set_config(_Key, _Value) -> 295 | ok. 296 | 297 | 298 | try_and_start(Port) -> 299 | erlang:port_control(Port, ?CMD_START, <<>>), 300 | receive 301 | innostore_ok -> 302 | {ok, Port}; 303 | {innostore_error, starting} -> 304 | timer:sleep(50), 305 | try_and_start(Port); 306 | {innostore_error, Reason} -> 307 | {error, Reason} 308 | end. 309 | 310 | list_keystores_loop(Acc) -> 311 | receive 312 | {innostore_table_name, "innokeystore/" ++ Table} -> 313 | list_keystores_loop([Table | Acc]); 314 | innostore_ok -> 315 | lists:reverse(Acc); 316 | {innostore_error, Reason} -> 317 | {error, Reason} 318 | end. 319 | 320 | 321 | cursor_open(Store) -> 322 | erlang:port_control(Store#store.port, ?CMD_CURSOR_OPEN, <<(Store#store.table_id)/binary>>), 323 | receive 324 | innostore_ok -> 325 | ok; 326 | {innostore_error, Reason} -> 327 | {error, Reason} 328 | end. 329 | 330 | cursor_move(Direction, Content, Store) -> 331 | erlang:port_control(Store#store.port, ?CMD_CURSOR_MOVE, <>), 332 | receive 333 | {innostore_ok, Key} -> 334 | {ok, Key}; 335 | {innostore_ok, Key, Value} -> 336 | {ok, Key, Value}; 337 | {innostore_error, Reason} -> 338 | {error, Reason} 339 | end. 340 | 341 | cursor_close(Store) -> 342 | erlang:port_control(Store#store.port, ?CMD_CURSOR_CLOSE, <<>>), 343 | receive 344 | innostore_ok -> 345 | ok; 346 | {innostore_error, Reason} -> 347 | {error, Reason} 348 | end. 349 | 350 | fold(Fun, Acc0, Content, Store) -> 351 | case cursor_open(Store) of 352 | ok -> 353 | case fold_loop(?CURSOR_FIRST, Content, Fun, Acc0, Store) of 354 | {ok, Acc} -> 355 | cursor_close(Store), 356 | Acc; 357 | {error, Reason} -> 358 | {error, Reason} 359 | end; 360 | {error, Reason} -> 361 | {error, Reason} 362 | end. 363 | 364 | fold_loop(Direction, Content, Fun, Acc, Store) -> 365 | case cursor_move(Direction, Content, Store) of 366 | {ok, eof} -> 367 | {ok, Acc}; 368 | {ok, Key} -> 369 | Acc1 = Fun(Key, Acc), 370 | fold_loop(?CURSOR_NEXT, Content, Fun, Acc1, Store); 371 | {ok, Key, Value} -> 372 | Acc1 = Fun(Key, Value, Acc), 373 | fold_loop(?CURSOR_NEXT, Content, Fun, Acc1, Store); 374 | {error, Reason} -> 375 | {error, Reason} 376 | end. 377 | 378 | 379 | 380 | %% 381 | %% Configuration type information. Extracted from api/api0cfg.c in inno distribution. 382 | %% 383 | config_types() -> 384 | [{adaptive_hash_index, bool}, 385 | {adaptive_flushing, bool}, 386 | {additional_mem_pool_size, integer}, 387 | {autoextend_increment, integer}, 388 | {buffer_pool_size, integer}, 389 | {checksums, bool}, 390 | {data_file_path, string}, 391 | {data_home_dir, string}, 392 | {doublewrite, bool}, 393 | {error_log, string}, 394 | {file_format, string}, 395 | {file_io_threads, integer}, 396 | {file_per_table, bool}, 397 | {flush_log_at_trx_commit, integer}, 398 | {flush_method, string}, 399 | {force_recovery, integer}, 400 | {io_capacity, integer}, 401 | {lock_wait_timeout, integer}, 402 | {log_buffer_size, integer}, 403 | {log_file_size, integer}, 404 | {log_files_in_group, integer}, 405 | {log_group_home_dir, string}, 406 | {max_dirty_pages_pct, integer}, 407 | {max_purge_lag, integer}, 408 | {lru_old_blocks_pct, integer}, 409 | {lru_block_access_recency, integer}, 410 | {open_files, integer}, 411 | {read_io_threads, integer}, 412 | {write_io_threads, integer}, 413 | {print_verbose_log, bool}, 414 | {rollback_on_timeout, bool}, 415 | {status_file, bool}, 416 | {sync_spin_loops, integer}, 417 | {use_sys_malloc, bool}, 418 | {version, string}]. 419 | 420 | status_names() -> 421 | [%% IO system related 422 | read_req_pending, 423 | write_req_pending, 424 | fsync_req_pending, 425 | write_req_done, 426 | read_req_done, 427 | fsync_req_done, 428 | bytes_total_written, 429 | bytes_total_read, 430 | 431 | %% Buffer pool related 432 | buffer_pool_current_size, 433 | buffer_pool_data_pages, 434 | buffer_pool_dirty_pages, 435 | buffer_pool_misc_pages, 436 | buffer_pool_free_pages, 437 | buffer_pool_read_reqs, 438 | buffer_pool_reads, 439 | buffer_pool_waited_for_free, 440 | buffer_pool_pages_flushed, 441 | buffer_pool_write_reqs, 442 | buffer_pool_total_pages, 443 | buffer_pool_pages_read, 444 | buffer_pool_pages_written, 445 | 446 | %% Double write buffer related 447 | double_write_pages_written, 448 | double_write_invoked, 449 | 450 | %% Log related 451 | log_buffer_slot_waits, 452 | log_write_reqs, 453 | log_write_flush_count, 454 | log_bytes_written, 455 | log_fsync_req_done, 456 | log_write_req_pending, 457 | log_fsync_req_pending, 458 | 459 | %% Lock related 460 | lock_row_waits, 461 | lock_row_waiting, 462 | lock_total_wait_time_in_secs, 463 | lock_wait_time_avg_in_secs, 464 | lock_max_wait_time_in_secs, 465 | 466 | %% Row operations 467 | row_total_read, 468 | row_total_inserted, 469 | row_total_updated, 470 | row_total_deleted, 471 | 472 | %% Miscellaneous 473 | page_size, 474 | have_atomic_builtins]. 475 | 476 | %% 477 | %% Encode configuration setting, based on type for passing through to inno api 478 | %% 479 | config_encode(integer, _Key, Value) -> 480 | case erlang:system_info(wordsize) of 481 | 4 -> <>; 482 | 8 -> <> 483 | end; 484 | config_encode(bool, Key, true) -> 485 | config_encode(integer, Key, 1); 486 | config_encode(bool, Key, false) -> 487 | config_encode(integer, Key, 0); 488 | config_encode(string, data_home_dir, Value) -> 489 | %% Make sure that the last character is a path separator 490 | CleanedUp = filename:absname(Value) ++ "/", 491 | <<(list_to_binary(CleanedUp))/binary, 0:8>>; 492 | config_encode(string, _Key, Value) -> 493 | <<(list_to_binary(Value))/binary, 0:8>>. 494 | 495 | %% Work out what format to use - fallback to compact if 496 | %% none configured. 497 | config_format() -> 498 | case application:get_env(innostore, format) of 499 | {ok, Format} -> 500 | Format; 501 | _ -> 502 | compact 503 | end. 504 | 505 | format_encode(redundant) -> 506 | ?FORMAT_REDUNDANT; 507 | format_encode(compact) -> 508 | ?FORMAT_COMPACT; 509 | format_encode(dynamic) -> 510 | ?FORMAT_DYNAMIC; 511 | format_encode(compressed) -> 512 | ?FORMAT_COMPRESSED. 513 | 514 | %% =================================================================== 515 | %% EUnit tests 516 | %% =================================================================== 517 | -ifdef(TEST). 518 | 519 | 520 | innostore_test_() -> 521 | [ 522 | %% These tests all run under the same process and only load the 523 | %% driver once/run which means it only runs ib_startup once 524 | {spawn, [ 525 | {"startup", ?_test(begin 526 | {ok, Port} = connect(), 527 | true = is_started(Port), 528 | {ok, Port2} = connect(), 529 | true = is_started(Port2) 530 | end)}, 531 | {"status", ?_test(begin 532 | {ok, Port} = connect(), 533 | ?assertEqual(16384, status(page_size, Port)) 534 | end)}, 535 | 536 | {"roundtrip", ?_test(ok = roundtrip_test_op(?COMPRESSION_NONE))}, 537 | 538 | {"list_tables", ?_test(begin 539 | {ok, Port} = connect_reset(), 540 | {ok, _} = open_keystore(foobar, Port), 541 | {ok, _} = open_keystore(barbaz, Port), 542 | {ok, _} = open_keystore(bazbaz, Port), 543 | ["barbaz", "bazbaz", "foobar"] = 544 | lists:sort(list_keystores(Port)) 545 | end)}, 546 | 547 | {"table_is_empty", ?_test(begin 548 | {ok, Port} = connect_reset(), 549 | {ok, Store} = open_keystore(foobar, Port), 550 | ok = ?MODULE:put(<<"abc">>, <<"def">>, Store), 551 | false = is_keystore_empty(foobar, Port), 552 | ok = ?MODULE:delete(<<"abc">>, Store), 553 | true = is_keystore_empty(foobar, Port), 554 | true = is_keystore_empty(nosuchtable, Port) 555 | end)}, 556 | 557 | {"bigkey", ?_test(begin 558 | {ok, Port} = connect_reset(), 559 | {ok, Store} = open_keystore(foobar, Port), 560 | Key = list_to_binary(lists:duplicate(256, "x")), 561 | {error, key_exceeds_255_bytes} = ?MODULE:put(Key, <<"abc">>, Store), 562 | 563 | Key2 = list_to_binary(lists:duplicate(153, "x")), 564 | ok = ?MODULE:put(Key2, <<"abc">>, Store) 565 | end)} 566 | ]}, 567 | 568 | %% Run error_log testing in a new process - needs to set up the log file 569 | %% as the driver is loaded 570 | {spawn, {"error_log", {timeout, 60, ?_test( 571 | begin 572 | LogFile = "innodb_eunit.log", 573 | file:delete(LogFile), 574 | false = filelib:is_regular(LogFile), 575 | false = innostore_loaded(), 576 | application:set_env(innostore, error_log, LogFile), 577 | {ok, Port} = connect(), 578 | true = is_started(Port), 579 | true = filelib:is_regular(LogFile), 580 | application:unset_env(innostore, error_log) 581 | end)}}} 582 | ]. 583 | 584 | connect_reset() -> 585 | {ok, Port} = connect(), 586 | [ok = drop_keystore(T, Port) || T <- list_keystores(Port)], 587 | {ok, Port}. 588 | 589 | roundtrip_test_op(Compression) -> 590 | {ok, Port} = connect_reset(), 591 | {ok, Store} = open_keystore(test, Port), 592 | S2 = Store#store { compression = Compression }, 593 | ok = ?MODULE:put(<<"key1">>, <<"value1">>, S2), 594 | {ok, <<"value1">>} = ?MODULE:get(<<"key1">>, S2), 595 | ok = ?MODULE:delete(<<"key1">>, S2), 596 | {ok, not_found} = ?MODULE:get(<<"key1">>, S2), 597 | ok. 598 | 599 | innostore_loaded() -> 600 | {ok, D} = erl_ddll:loaded_drivers(), 601 | case lists:member("innostore_drv", D) of 602 | true -> 603 | io:format("~p\n", [erl_ddll:driver_info(innostore_drv)]), 604 | true; 605 | false -> 606 | false 607 | end. 608 | 609 | -endif. 610 | -------------------------------------------------------------------------------- /src/innostore_riak.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% innostore: Simple Erlang API to Embedded Inno DB 4 | %% 5 | %% Copyright (c) 2009 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% innostore is free software: you can redistribute it and/or modify 8 | %% it under the terms of the GNU General Public License as published by 9 | %% the Free Software Foundation, either version 2 of the License, or 10 | %% (at your option) any later version. 11 | %% 12 | %% innostore is distributed in the hope that it will be useful, 13 | %% but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | %% GNU General Public License for more details. 16 | %% 17 | %% You should have received a copy of the GNU General Public License 18 | %% along with innostore. If not, see . 19 | %% 20 | %% ------------------------------------------------------------------- 21 | -module(innostore_riak). 22 | 23 | -define(OVERRIDE_MODULE, 1). 24 | -include("riak_kv_innostore_backend.erl"). 25 | -------------------------------------------------------------------------------- /src/riak_kv_innostore_backend.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% innostore: Simple Erlang API to Embedded Inno DB 4 | %% 5 | %% Copyright (c) 2009 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% innostore is free software: you can redistribute it and/or modify 8 | %% it under the terms of the GNU General Public License as published by 9 | %% the Free Software Foundation, either version 2 of the License, or 10 | %% (at your option) any later version. 11 | %% 12 | %% innostore is distributed in the hope that it will be useful, 13 | %% but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | %% GNU General Public License for more details. 16 | %% 17 | %% You should have received a copy of the GNU General Public License 18 | %% along with innostore. If not, see . 19 | %% 20 | %% ------------------------------------------------------------------- 21 | -ifndef(OVERRIDE_MODULE). 22 | -module(riak_kv_innostore_backend). 23 | -endif. 24 | 25 | -author('Dave Smith '). 26 | 27 | %% KV Backend API 28 | -export([api_version/0, 29 | capabilities/1, 30 | capabilities/2, 31 | start/2, 32 | stop/1, 33 | get/3, 34 | put/5, 35 | delete/4, 36 | drop/1, 37 | fold_buckets/4, 38 | fold_keys/4, 39 | fold_objects/4, 40 | is_empty/1, 41 | status/1, 42 | callback/3]). 43 | 44 | -ifdef(TEST). 45 | -include_lib("eunit/include/eunit.hrl"). 46 | -endif. 47 | 48 | -define(API_VERSION, 1). 49 | -define(CAPABILITIES, [async_fold]). 50 | 51 | -record(state, { partition_str, 52 | port }). 53 | 54 | -opaque(state() :: #state{}). 55 | -type config() :: [{atom(), term()}]. 56 | -type bucket() :: binary(). 57 | -type key() :: binary(). 58 | -type fold_buckets_fun() :: fun((binary(), any()) -> any() | no_return()). 59 | -type fold_keys_fun() :: fun((binary(), binary(), any()) -> any() | 60 | no_return()). 61 | -type fold_objects_fun() :: fun((binary(), binary(), term(), any()) -> 62 | any() | 63 | no_return()). 64 | 65 | %% =================================================================== 66 | %% Public API 67 | %% =================================================================== 68 | 69 | %% @doc Return the major version of the 70 | %% current API. 71 | -spec api_version() -> {ok, integer()}. 72 | api_version() -> 73 | {ok, ?API_VERSION}. 74 | 75 | %% @doc Return the capabilities of the backend. 76 | -spec capabilities(state()) -> {ok, [atom()]}. 77 | capabilities(_) -> 78 | {ok, ?CAPABILITIES}. 79 | 80 | %% @doc Return the capabilities of the backend. 81 | -spec capabilities(riak_object:bucket(), state()) -> {ok, [atom()]}. 82 | capabilities(_, _) -> 83 | {ok, ?CAPABILITIES}. 84 | 85 | %% @doc Start the innostore backend 86 | -spec start(integer(), config()) -> {ok, state()} | {error, term()}. 87 | start(Partition, _Config) -> 88 | case innostore:connect() of 89 | {ok, Port} -> 90 | PartitionStr = <<"_", (list_to_binary(integer_to_list(Partition)))/binary>>, 91 | {ok, #state { partition_str = PartitionStr, port = Port }}; 92 | {error, Reason} -> 93 | {error, Reason} 94 | end. 95 | 96 | %% @doc Stop the innostore backend 97 | -spec stop(state()) -> ok. 98 | stop(State) -> 99 | innostore:disconnect(State#state.port). 100 | 101 | %% @doc Retrieve an object from the innostore backend 102 | -spec get(bucket(), key(), state()) -> 103 | {ok, any(), state()} | 104 | {ok, not_found, state()} | 105 | {error, term(), state()}. 106 | get(Bucket, Key, #state{partition_str=Partition, 107 | port=Port}=State) -> 108 | case innostore:get(Key, keystore(Bucket, Partition, Port)) of 109 | {ok, not_found} -> 110 | {error, not_found, State}; 111 | {ok, Value} -> 112 | {ok, Value, State}; 113 | {error, Reason} -> 114 | {error, Reason, State} 115 | end. 116 | 117 | %% @doc Insert an object into the innostore backend. 118 | %% NOTE: The innostore backend does not currently support 119 | %% secondary indexing and the _IndexSpecs parameter 120 | %% is ignored. 121 | -type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. 122 | -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> 123 | {ok, state()} | 124 | {error, term(), state()}. 125 | put(Bucket, Key, _IndexSpecs, Value, #state{partition_str=Partition, 126 | port=Port}=State) -> 127 | KeyStore = keystore(Bucket, Partition, Port), 128 | case innostore:put(Key, Value, KeyStore) of 129 | ok -> 130 | {ok, State}; 131 | {error, Reason} -> 132 | {error, Reason, State} 133 | end. 134 | 135 | %% @doc Delete an object from the innostore backend 136 | %% NOTE: The innostore backend does not currently support 137 | %% secondary indexing and the _IndexSpecs parameter 138 | %% is ignored. 139 | -spec delete(bucket(), key(), [index_spec()], state()) -> 140 | {ok, state()} | 141 | {error, term(), state()}. 142 | delete(Bucket, Key, _IndexSpecs, #state{partition_str=Partition, 143 | port=Port}=State) -> 144 | KeyStore = keystore(Bucket, Partition, Port), 145 | case innostore:delete(Key, KeyStore) of 146 | ok -> 147 | {ok, State}; 148 | {error, Reason} -> 149 | {error, Reason, State} 150 | end. 151 | 152 | %% @doc Fold over all the buckets. 153 | -spec fold_buckets(fold_buckets_fun(), 154 | any(), 155 | [], 156 | state()) -> {ok, any()} | {async, fun()} | {error, term()}. 157 | fold_buckets(FoldBucketsFun, Acc, Opts, #state{partition_str=Partition, 158 | port=Port}) -> 159 | FoldFun = fold_buckets_fun(FoldBucketsFun), 160 | Suffix = binary_to_list(Partition), 161 | FilterFun = 162 | fun(KeyStore, Acc1) -> 163 | case lists:suffix(Suffix, KeyStore) of 164 | true -> 165 | Bucket = bucket_from_tablename(KeyStore), 166 | FoldFun(Bucket, Acc1); 167 | false -> 168 | Acc1 169 | end 170 | end, 171 | case lists:member(async_fold, Opts) of 172 | true -> 173 | BucketFolder = 174 | fun() -> 175 | case get_port() of 176 | {ok, Port1} -> 177 | FoldResults = 178 | lists:foldl(FilterFun, 179 | Acc, 180 | innostore:list_keystores(Port1)), 181 | FoldResults; 182 | {error, Reason} -> 183 | {error, Reason} 184 | end 185 | end, 186 | {async, BucketFolder}; 187 | false -> 188 | FoldResults = lists:foldl(FilterFun, 189 | Acc, 190 | innostore:list_keystores(Port)), 191 | {ok, FoldResults} 192 | end. 193 | 194 | %% @doc Fold over all the keys for one or all buckets. 195 | -spec fold_keys(fold_keys_fun(), 196 | any(), 197 | [{atom(), term()}], 198 | state()) -> {ok, term()} | {error, term()}. 199 | fold_keys(FoldKeysFun, Acc, Opts, #state{partition_str=Partition, 200 | port=Port}) -> 201 | Bucket = proplists:get_value(bucket, Opts), 202 | case lists:member(async_fold, Opts) of 203 | true -> 204 | KeyFolder = async_key_folder(Bucket, FoldKeysFun, Acc, Partition), 205 | {async, KeyFolder}; 206 | false -> 207 | FoldResults = sync_key_fold(Bucket, FoldKeysFun, Acc, Partition, Port), 208 | {ok, FoldResults} 209 | end. 210 | 211 | %% @doc Fold over all the objects for one or all buckets. 212 | -spec fold_objects(fold_objects_fun(), 213 | any(), 214 | [{atom(), term()}], 215 | state()) -> {ok, any()} | {error, term()}. 216 | fold_objects(FoldObjectsFun, Acc, Opts, #state{partition_str=Partition, 217 | port=Port}) -> 218 | Bucket = proplists:get_value(bucket, Opts), 219 | case lists:member(async_fold, Opts) of 220 | true -> 221 | KeyFolder = async_object_folder(Bucket, FoldObjectsFun, Acc, Partition), 222 | {async, KeyFolder}; 223 | false -> 224 | FoldResults = sync_object_fold(Bucket, FoldObjectsFun, Acc, Partition, Port), 225 | {ok, FoldResults} 226 | end. 227 | 228 | %% @doc Delete all objects from this innostore backend 229 | -spec drop(state()) -> {ok, state()} | {error, term(), state()}. 230 | drop(#state{partition_str=Partition, 231 | port=Port}=State) -> 232 | KeyStores = list_keystores(Partition, Port), 233 | [innostore:drop_keystore(KeyStore, Port) || KeyStore <- KeyStores], 234 | {ok, State}. 235 | 236 | %% @doc Returns true if this innostore backend contains any 237 | %% non-tombstone values; otherwise returns false. 238 | -spec is_empty(state()) -> boolean() | {error, term()}. 239 | is_empty(#state{partition_str=Partition, 240 | port=Port}) -> 241 | lists:all(fun(I) -> I end, 242 | [innostore:is_keystore_empty(B, Port) || 243 | B <- list_keystores(Partition, Port)]). 244 | 245 | %% @doc Get the status information for this innostore backend 246 | -spec status(state()) -> [{atom(), term()}]. 247 | status(#state{port=Port}) -> 248 | Status = innostore:status(Port), 249 | format_status(Status). 250 | 251 | %% Ignore callbacks we do not know about - may be in multi backend 252 | callback(_Ref, _Msg, State) -> 253 | {ok, State}. 254 | 255 | %% =================================================================== 256 | %% Internal functions 257 | %% =================================================================== 258 | 259 | %% @private 260 | get_port() -> 261 | case erlang:get(inno_port) of 262 | undefined -> 263 | case innostore:connect() of 264 | {ok, Port} -> 265 | erlang:put(inno_port, Port), 266 | {ok, Port}; 267 | Error -> 268 | Error 269 | end; 270 | Port -> 271 | {ok, Port} 272 | end. 273 | 274 | %% @private 275 | keystore(Bucket, Partition, Port) -> 276 | KeyStoreId = <>, 277 | case erlang:get({innostore, KeyStoreId}) of 278 | undefined -> 279 | {ok, KeyStore} = innostore:open_keystore(KeyStoreId, Port), 280 | erlang:put({innostore, KeyStoreId}, KeyStore), 281 | KeyStore; 282 | KeyStore -> 283 | KeyStore 284 | end. 285 | 286 | %% @private 287 | %% Return a function to fold over the buckets on this backend 288 | fold_buckets_fun(FoldBucketsFun) -> 289 | fun(Bucket, Acc) -> 290 | FoldBucketsFun(Bucket, Acc) 291 | end. 292 | 293 | %% @private 294 | %% Return a function to synchronously fold over keys on this backend 295 | async_key_folder(undefined, FoldFun, Acc, Partition) -> 296 | fun() -> 297 | case get_port() of 298 | {ok, Port} -> 299 | Buckets = list_buckets(Partition, Port), 300 | %% Fold over all keys in all buckets 301 | FoldResults = fold_all_keys(Buckets, 302 | Acc, 303 | FoldFun, 304 | Partition, 305 | Port), 306 | FoldResults; 307 | {error, Reason} -> 308 | {error, Reason} 309 | end 310 | end; 311 | async_key_folder(Bucket, FoldFun, Acc, Partition) -> 312 | FoldKeysFun = fold_keys_fun(FoldFun, Bucket), 313 | fun() -> 314 | case get_port() of 315 | {ok, Port} -> 316 | KeyStore = keystore(Bucket, Partition, Port), 317 | FoldResults = 318 | innostore:fold_keys(FoldKeysFun, Acc, KeyStore), 319 | FoldResults; 320 | {error, Reason} -> 321 | {error, Reason} 322 | end 323 | end. 324 | 325 | %% @private 326 | %% Return a function to synchronously fold over keys on this backend 327 | sync_key_fold(undefined, FoldFun, Acc, Partition, Port) -> 328 | Buckets = list_buckets(Partition, Port), 329 | %% Fold over all keys in all buckets 330 | fold_all_keys(Buckets, 331 | Acc, 332 | FoldFun, 333 | Partition, 334 | Port); 335 | sync_key_fold(Bucket, FoldFun, Acc, Partition, Port) -> 336 | FoldKeysFun = fold_keys_fun(FoldFun, Bucket), 337 | KeyStore = keystore(Bucket, Partition, Port), 338 | innostore:fold_keys(FoldKeysFun, Acc, KeyStore). 339 | 340 | %% @private 341 | fold_keys_fun(FoldKeysFun, Bucket) -> 342 | fun(Key, Acc) -> 343 | FoldKeysFun(Bucket, Key, Acc) 344 | end. 345 | 346 | %% @private 347 | %% Return a function to synchronously fold over objects on this backend 348 | sync_object_fold(undefined, FoldFun, Acc, Partition, Port) -> 349 | Buckets = list_buckets(Partition, Port), 350 | %% Fold over all keys in all buckets 351 | fold_all_objects(Buckets, 352 | Acc, 353 | FoldFun, 354 | Partition, 355 | Port); 356 | sync_object_fold(Bucket, FoldFun, Acc, Partition, Port) -> 357 | FoldObjectsFun = fold_objects_fun(FoldFun, Bucket), 358 | KeyStore = keystore(Bucket, Partition, Port), 359 | innostore:fold_keys(FoldObjectsFun, Acc, KeyStore). 360 | 361 | %% @private 362 | %% Return a function to synchronously fold over objects on this backend 363 | async_object_folder(undefined, FoldFun, Acc, Partition) -> 364 | fun() -> 365 | case get_port() of 366 | {ok, Port} -> 367 | Buckets = list_buckets(Partition, Port), 368 | %% Fold over all objects in all buckets 369 | FoldResults = fold_all_objects(Buckets, 370 | Acc, 371 | FoldFun, 372 | Partition, 373 | Port), 374 | FoldResults; 375 | {error, Reason} -> 376 | {error, Reason} 377 | end 378 | end; 379 | async_object_folder(Bucket, FoldFun, Acc, Partition) -> 380 | FoldObjectsFun = fold_objects_fun(FoldFun, Bucket), 381 | fun() -> 382 | case get_port() of 383 | {ok, Port} -> 384 | KeyStore = keystore(Bucket, Partition, Port), 385 | FoldResults = innostore:fold(FoldObjectsFun, Acc, KeyStore), 386 | FoldResults; 387 | {error, Reason} -> 388 | {error, Reason} 389 | end 390 | end. 391 | 392 | %% @private 393 | %% Return a function to fold over keys on this backend 394 | fold_objects_fun(FoldObjectsFun, Bucket) -> 395 | fun(Key, Value, Acc) -> 396 | FoldObjectsFun(Bucket, Key, Value, Acc) 397 | end. 398 | 399 | %% @private 400 | list_buckets(Partition, Port) -> 401 | Suffix = binary_to_list(Partition), 402 | [bucket_from_tablename(KeyStore) || KeyStore <- innostore:list_keystores(Port), 403 | lists:suffix(Suffix, KeyStore) == true]. 404 | 405 | %% @private 406 | list_keystores(Partition, Port) -> 407 | Suffix = binary_to_list(Partition), 408 | [KeyStore || KeyStore <- innostore:list_keystores(Port), 409 | lists:suffix(Suffix, KeyStore) == true]. 410 | 411 | %% @private 412 | fold_all_keys([], Acc, _, _Partition, _Port) -> 413 | Acc; 414 | fold_all_keys([Bucket | RestBuckets], Acc, FoldKeysFun, Partition, Port) -> 415 | KeyStore = keystore(Bucket, Partition, Port), 416 | FoldFun = fold_keys_fun(FoldKeysFun, Bucket), 417 | case innostore:fold_keys(FoldFun, Acc, KeyStore) of 418 | {error, Reason} -> 419 | {error, Reason}; 420 | Acc1 -> 421 | fold_all_keys(RestBuckets, Acc1, FoldKeysFun, Partition, Port) 422 | end. 423 | 424 | %% @private 425 | fold_all_objects([], Acc, _, _Partition, _Port) -> 426 | Acc; 427 | fold_all_objects([Bucket | RestBuckets], Acc, FoldObjectsFun, Partition, Port) -> 428 | KeyStore = keystore(Bucket, Partition, Port), 429 | FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), 430 | case innostore:fold(FoldFun, Acc, KeyStore) of 431 | {error, Reason} -> 432 | {error, Reason}; 433 | Acc1 -> 434 | fold_all_objects(RestBuckets, Acc1, FoldObjectsFun, Partition, Port) 435 | end. 436 | 437 | %% @private 438 | bucket_from_tablename(TableName) -> 439 | {match, [Name]} = re:run(TableName, "(.*)_\\d+", [{capture, all_but_first, binary}]), 440 | Name. 441 | 442 | %% @private 443 | format_status([]) -> ok; 444 | format_status([{K,V}|T]) -> 445 | io:format("~p: ~p~n", [K,V]), 446 | format_status(T). 447 | 448 | %% =================================================================== 449 | %% EUnit tests 450 | %% =================================================================== 451 | -ifdef(TEST). 452 | 453 | standard_test_() -> 454 | Config = [ 455 | {data_home_dir, "./test/innodb-backend"}, 456 | {log_group_home_dir, "./test/innodb-backend"}, 457 | {buffer_pool_size, 2147483648} 458 | ], 459 | {spawn, 460 | [ 461 | {setup, 462 | fun() -> setup(Config) end, 463 | fun cleanup/1, 464 | fun(X) -> 465 | [basic_store_and_fetch(X), 466 | fold_buckets(X), 467 | fold_keys(X), 468 | delete_object(X), 469 | fold_objects(X), 470 | empty_check(X) 471 | ] 472 | end 473 | }]}. 474 | 475 | basic_store_and_fetch(State) -> 476 | {"basic store and fetch test", 477 | fun() -> 478 | [ 479 | ?_assertMatch({ok, _}, 480 | ?MODULE:put(<<"b1">>, <<"k1">>, [], <<"v1">>, State)), 481 | ?_assertMatch({ok, _}, 482 | ?MODULE:put(<<"b2">>, <<"k2">>, [], <<"v2">>, State)), 483 | ?_assertMatch({ok,<<"v2">>, _}, 484 | ?MODULE:get(<<"b2">>, <<"k2">>, State)), 485 | ?_assertMatch({error, not_found, _}, 486 | ?MODULE:get(<<"b1">>, <<"k3">>, State)) 487 | ] 488 | end 489 | }. 490 | 491 | fold_buckets(State) -> 492 | {"bucket folding test", 493 | fun() -> 494 | FoldBucketsFun = 495 | fun(Bucket, Acc) -> 496 | [Bucket | Acc] 497 | end, 498 | 499 | ?_assertEqual([<<"b1">>, <<"b2">>], 500 | begin 501 | {ok, Buckets1} = 502 | ?MODULE:fold_buckets(FoldBucketsFun, 503 | [], 504 | [], 505 | State), 506 | lists:sort(Buckets1) 507 | end) 508 | end 509 | }. 510 | 511 | fold_keys(State) -> 512 | {"key folding test", 513 | fun() -> 514 | FoldKeysFun = 515 | fun(Bucket, Key, Acc) -> 516 | [{Bucket, Key} | Acc] 517 | end, 518 | FoldKeysFun1 = 519 | fun(_Bucket, Key, Acc) -> 520 | [Key | Acc] 521 | end, 522 | FoldKeysFun2 = 523 | fun(Bucket, Key, Acc) -> 524 | case Bucket =:= <<"b1">> of 525 | true -> 526 | [Key | Acc]; 527 | false -> 528 | Acc 529 | end 530 | end, 531 | FoldKeysFun3 = 532 | fun(Bucket, Key, Acc) -> 533 | case Bucket =:= <<"b1">> of 534 | true -> 535 | Acc; 536 | false -> 537 | [Key | Acc] 538 | end 539 | end, 540 | [ 541 | ?_assertEqual([{<<"b1">>, <<"k1">>}, {<<"b2">>, <<"k2">>}], 542 | begin 543 | {ok, Keys1} = 544 | ?MODULE:fold_keys(FoldKeysFun, 545 | [], 546 | [], 547 | State), 548 | lists:sort(Keys1) 549 | end), 550 | ?_assertEqual({ok, [<<"k1">>]}, 551 | ?MODULE:fold_keys(FoldKeysFun1, 552 | [], 553 | [{bucket, <<"b1">>}], 554 | State)), 555 | ?_assertEqual([<<"k2">>], 556 | ?MODULE:fold_keys(FoldKeysFun1, 557 | [], 558 | [{bucket, <<"b2">>}], 559 | State)), 560 | ?_assertEqual({ok, [<<"k1">>]}, 561 | ?MODULE:fold_keys(FoldKeysFun2, [], [], State)), 562 | ?_assertEqual({ok, [<<"k1">>]}, 563 | ?MODULE:fold_keys(FoldKeysFun2, 564 | [], 565 | [{bucket, <<"b1">>}], 566 | State)), 567 | ?_assertEqual({ok, [<<"k2">>]}, 568 | ?MODULE:fold_keys(FoldKeysFun3, [], [], State)), 569 | ?_assertEqual({ok, []}, 570 | ?MODULE:fold_keys(FoldKeysFun3, 571 | [], 572 | [{bucket, <<"b1">>}], 573 | State)) 574 | ] 575 | end 576 | }. 577 | 578 | delete_object(State) -> 579 | {"object deletion test", 580 | fun() -> 581 | [ 582 | ?_assertMatch({ok, _}, ?MODULE:delete(<<"b2">>, <<"k2">>, State)), 583 | ?_assertMatch({error, not_found, _}, 584 | ?MODULE:get(<<"b2">>, <<"k2">>, State)) 585 | ] 586 | end 587 | }. 588 | 589 | fold_objects(State) -> 590 | {"object folding test", 591 | fun() -> 592 | FoldKeysFun = 593 | fun(Bucket, Key, Acc) -> 594 | [{Bucket, Key} | Acc] 595 | end, 596 | FoldObjectsFun = 597 | fun(Bucket, Key, Value, Acc) -> 598 | [{{Bucket, Key}, Value} | Acc] 599 | end, 600 | [ 601 | ?_assertEqual([{<<"b1">>, <<"k1">>}], 602 | begin 603 | {ok, Keys} = 604 | ?MODULE:fold_keys(FoldKeysFun, 605 | [], 606 | [], 607 | State), 608 | lists:sort(Keys) 609 | end), 610 | 611 | ?_assertEqual([{{<<"b1">>,<<"k1">>}, <<"v1">>}], 612 | begin 613 | {ok, Objects1} = 614 | ?MODULE:fold_objects(FoldObjectsFun, 615 | [], 616 | [], 617 | State), 618 | lists:sort(Objects1) 619 | end), 620 | ?_assertMatch({ok, _}, 621 | ?MODULE:put(<<"b3">>, <<"k3">>, [], <<"v3">>, State)), 622 | ?_assertEqual([{{<<"b1">>,<<"k1">>},<<"v1">>}, 623 | {{<<"b3">>,<<"k3">>},<<"v3">>}], 624 | begin 625 | {ok, Objects} = 626 | ?MODULE:fold_objects(FoldObjectsFun, 627 | [], 628 | [], 629 | State), 630 | lists:sort(Objects) 631 | end) 632 | ] 633 | end 634 | }. 635 | 636 | empty_check(State) -> 637 | {"is_empty test", 638 | fun() -> 639 | [ 640 | ?_assertEqual(false, ?MODULE:is_empty(State)), 641 | ?_assertMatch({ok, _}, ?MODULE:delete(<<"b1">>,<<"k1">>, State)), 642 | ?_assertMatch({ok, _}, ?MODULE:delete(<<"b3">>,<<"k3">>, State)), 643 | ?_assertEqual(true, ?MODULE:is_empty(State)) 644 | ] 645 | end 646 | }. 647 | 648 | setup(Config) -> 649 | %% Start the backend 650 | {ok, S} = ?MODULE:start(42, Config), 651 | S. 652 | 653 | cleanup(S) -> 654 | ok = ?MODULE:stop(S), 655 | {ok, Port} = innostore:connect(), 656 | [ok = innostore:drop_keystore(T, Port) || T <- innostore:list_keystores(Port)], 657 | innostore:disconnect(Port), 658 | ok. 659 | 660 | -endif. 661 | --------------------------------------------------------------------------------