├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE.txt ├── README.md ├── al-v20.txt ├── pom.xml ├── redis └── Makefile └── src ├── main └── java │ └── net │ └── whitbeck │ └── rdbparser │ ├── AuxField.java │ ├── DoubleBytes.java │ ├── Entry.java │ ├── EntryType.java │ ├── Eof.java │ ├── IntSet.java │ ├── KeyValuePair.java │ ├── LazyList.java │ ├── ListpackList.java │ ├── Lzf.java │ ├── QuickList.java │ ├── QuickList2.java │ ├── RdbParser.java │ ├── ResizeDb.java │ ├── SelectDb.java │ ├── SortedSetAsListpack.java │ ├── SortedSetAsZipList.java │ ├── StringUtils.java │ ├── ValueType.java │ ├── ZipList.java │ ├── ZipMap.java │ └── package-info.java └── test └── java └── net └── whitbeck └── rdbparser └── RdbParserTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.classpath 3 | /.settings/ 4 | /.project 5 | workbench.xmi -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "redis/3.2.11"] 2 | path = redis/3.2.11 3 | url = https://github.com/antirez/redis.git 4 | [submodule "redis/2.8.24"] 5 | path = redis/2.8.24 6 | url = https://github.com/antirez/redis.git 7 | [submodule "redis/4.0.6"] 8 | path = redis/4.0.6 9 | url = https://github.com/antirez/redis.git 10 | [submodule "redis/5.0.14"] 11 | path = redis/5.0.14 12 | url = https://github.com/redis/redis.git 13 | [submodule "redis/6.2.1"] 14 | path = redis/6.2.1 15 | url = https://github.com/redis/redis.git 16 | [submodule "redis/7.0.11"] 17 | path = redis/7.0.11 18 | url = https://github.com/redis/redis.git 19 | [submodule "redis/7.2.4"] 20 | path = redis/7.2.4 21 | url = https://github.com/redis/redis.git 22 | [submodule "redis/7.4.1"] 23 | path = redis/7.4.1 24 | url = https://github.com/redis/redis.git 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | before_install: 3 | - sudo apt-get update -qq 4 | - sudo apt-get install -y wget build-essential git 5 | - git submodule update --init 6 | - make -C redis 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | License: 2 | 3 | Copyright (c) 2015-2021 John Whitbeck. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0.txt 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | Third-party Licenses: 18 | 19 | All third-party dependencies are listed in the pom.xml files. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A simple Redis RDB file parser for Java 2 | 3 | [![Build Status](https://travis-ci.org/jwhitbeck/java-rdb-parser.png)](https://travis-ci.org/jwhitbeck/java-rdb-parser.png) 4 | 5 | ## Overview 6 | 7 | A simple Java library for parsing [Redis](http://redis.io) RDB files. 8 | 9 | This library does the minimal amount of work to read entries (e.g. a new DB 10 | selector, or a key/value pair with an expire time) from an RDB file, mostly 11 | limiting itself to returning byte arrays or lists of byte arrays for keys and 12 | values. The caller is responsible for application-level decisions such as how to 13 | interpret the contents of the returned byte arrays or what types of objects to 14 | instantiate from them. 15 | 16 | For example, sorted sets and hashes are parsed as a flat list of value/score 17 | pairs and key/value pairs, respectively. Simple Redis values are parsed as a 18 | singleton. As expected, Redis lists and sets are parsed as lists of values. 19 | 20 | Furthermore, this library performs lazy decoding of the packed encodings 21 | (ZipMap, ZipList, Hashmap as ZipList, Sorted Set as ZipList, Intset, QuickList, 22 | and ListPack) such that those are only decoded when needed. This allows the 23 | caller to efficiently skip over these entries or defer their decoding to a 24 | worker thread. 25 | 26 | RDB files created by all versions of Redis through 7.4.x are supported (i.e., 27 | RDB versions 1 through 12). Some features, however, are not supported: 28 | 29 | - [Modules](https://redis.io/modules), introduced in RDB version 8 30 | - [Streams](https://redis.io/topics/streams-intro), introduced in RDB version 9. 31 | 32 | If you need these, please open an issue or a pull request. 33 | 34 | [Valkey](https://valkey.io/), an open source fork of Redis 7.2, uses the same 35 | RDB format as of 8.0.x, and this library can read those as well. 36 | 37 | To use this library, including the following dependency in your `pom.xml`. 38 | 39 | ```xml 40 | 41 | net.whitbeck 42 | rdb-parser 43 | 2.2.0 44 | 45 | ``` 46 | 47 | Javadocs are available at 48 | [javadoc.io/doc/net.whitbeck/rdb-parser/](http://www.javadoc.io/doc/net.whitbeck/rdb-parser/). 49 | 50 | ## Example usage 51 | 52 | Let's begin by creating a new Redis RDB dump file. 53 | 54 | Start a server in the background, connect a client to it, and flush all existing 55 | data. 56 | 57 | ``` 58 | $ redis-server & 59 | $ redis-cli 60 | 127.0.0.1:6379> flushall 61 | ``` 62 | 63 | Now let's create some data structures. Let's start with a simple key/value pair 64 | with an expire time. 65 | 66 | ``` 67 | 127.0.0.1:6379> set foo bar 68 | 127.0.0.1:6379> expire foo 3600 69 | ``` 70 | 71 | Then let's create a small hash and a sorted set. 72 | 73 | ``` 74 | 127.0.0.1:6379> hset myhash field1 val1 75 | 127.0.0.1:6379> hset myhash field2 val2 76 | 127.0.0.1:6379> zadd myset 1 one 2 two 2.5 two-point-five 77 | ``` 78 | 79 | Finally, let's save the dump to disk. This will create a `dump.rdb` file in the 80 | current directory. 81 | 82 | ``` 83 | 127.0.0.1:6379> save 84 | 127.0.0.1:6379> exit 85 | $ killall redis-server 86 | ``` 87 | 88 | Now let's see how to parse the `dump.rdb` file from Java. 89 | 90 | ```java 91 | import java.io.File; 92 | import net.whitbeck.rdbparser.*; 93 | 94 | public class RdbFilePrinter { 95 | 96 | public static void printRdbFile(File file) throws Exception { 97 | try (RdbParser parser = new RdbParser(file)) { 98 | Entry e; 99 | while ((e = parser.readNext()) != null) { 100 | switch (e.getType()) { 101 | 102 | case SELECT_DB: 103 | System.out.println("Processing DB: " + ((SelectDb)e).getId()); 104 | System.out.println("------------"); 105 | break; 106 | 107 | case EOF: 108 | System.out.print("End of file. Checksum: "); 109 | for (byte b : ((Eof)e).getChecksum()) { 110 | System.out.print(String.format("%02x", b & 0xff)); 111 | } 112 | System.out.println(); 113 | System.out.println("------------"); 114 | break; 115 | 116 | case KEY_VALUE_PAIR: 117 | System.out.println("Key value pair"); 118 | KeyValuePair kvp = (KeyValuePair)e; 119 | System.out.println("Key: " + new String(kvp.getKey(), "ASCII")); 120 | Long expireTime = kvp.getExpiretime(); 121 | if (expireTime != null) { 122 | System.out.println("Expire time (ms): " + expireTime); 123 | } 124 | System.out.println("Value type: " + kvp.getValueType()); 125 | System.out.print("Values: "); 126 | for (byte[] val : kvp.getValues()) { 127 | System.out.print(new String(val, "ASCII") + " "); 128 | } 129 | System.out.println(); 130 | System.out.println("------------"); 131 | break; 132 | } 133 | } 134 | } 135 | } 136 | } 137 | ``` 138 | 139 | Call this function on the `dump.rdb` file. The output will look like: 140 | 141 | ``` 142 | Processing DB: 0 143 | ------------ 144 | Key value pair 145 | Key: myset 146 | Value type: SORTED_SET_AS_ZIPLIST 147 | Values: one 1 two 2 two-point-five 2.5 148 | ------------ 149 | Key value pair 150 | Key: myhash 151 | Value type: HASHMAP_AS_ZIPLIST 152 | Values: field1 val1 field2 val2 153 | ------------ 154 | Key value pair 155 | Key: foo 156 | Expire time (ms): 1451518660934 157 | Value type: VALUE 158 | Values: bar 159 | ------------ 160 | End of file. Checksum: 157e40ad49ef13f6 161 | ------------ 162 | ``` 163 | 164 | ## References 165 | 166 | As of November 2024, the most recent RDB format version is 12. The source of 167 | truth is the [rdb.h][] file in the [Redis repo][]. The following resources 168 | provide a good overview of the RDB format. 169 | 170 | - [RDB file format](http://rdb.fnordig.de/file_format.html) (up to version 7). 171 | - [RDB file format (redis-rdb-tools)](https://github.com/sripathikrishnan/redis-rdb-tools/wiki/Redis-RDB-Dump-File-Format) 172 | - [RDB version history (redis-rdb-tools)](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_Version_History.textile) 173 | 174 | [rdb.h]: https://github.com/redis/redis/blob/unstable/src/rdb.h 175 | [Redis repo]: https://github.com/redis/redis 176 | -------------------------------------------------------------------------------- /al-v20.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | net.whitbeck 5 | rdb-parser 6 | 2.2.0 7 | 8 | ${project.groupId}:${project.artifactId} 9 | A simple Redis RDB file parser for Java 10 | https://github.com/jwhitbeck/java-rdb-parser 11 | 12 | 13 | The Apache License, Version 2.0 14 | http://www.apache.org/licenses/LICENSE-2.0.txt 15 | 16 | 17 | 18 | 19 | John Whitbeck 20 | john@whitbeck.net 21 | 22 | 23 | 24 | scm:git:https://github.com/jwhitbeck/java-rdb-parser.git 25 | scm:git:git@github.com:jwhitbeck/java-rdb-parser.git 26 | https://github.com/jwhitbeck/java-rdb-parser 27 | 28 | 29 | UTF-8 30 | google_checks.xml 31 | 32 | 33 | 34 | 35 | junit 36 | junit 37 | 4.13.1 38 | test 39 | 40 | 41 | 42 | redis.clients 43 | jedis 44 | 5.2.0 45 | jar 46 | test 47 | 48 | 49 | 50 | 51 | 52 | 53 | org.apache.maven.plugins 54 | maven-compiler-plugin 55 | 2.1 56 | 57 | 1.7 58 | 1.7 59 | 60 | 61 | 62 | 63 | 64 | org.sonatype.central 65 | central-publishing-maven-plugin 66 | 0.6.0 67 | true 68 | 69 | central 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | org.codehaus.mojo 80 | findbugs-maven-plugin 81 | 3.0.5 82 | 83 | false 84 | 85 | 86 | 87 | 88 | 89 | org.apache.maven.plugins 90 | maven-checkstyle-plugin 91 | 3.3.0 92 | 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-pmd-plugin 98 | 3.21.0 99 | 100 | 101 | 102 | 103 | 104 | 105 | release 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-gpg-plugin 112 | 1.5 113 | 114 | 115 | sign-artifacts 116 | verify 117 | 118 | sign 119 | 120 | 121 | 122 | 123 | 124 | 125 | org.apache.maven.plugins 126 | maven-source-plugin 127 | 2.2.1 128 | 129 | 130 | attach-sources 131 | 132 | jar-no-fork 133 | 134 | 135 | 136 | 137 | 138 | 139 | org.apache.maven.plugins 140 | maven-javadoc-plugin 141 | 2.9.1 142 | 143 | 144 | attach-javadocs 145 | 146 | jar 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | -------------------------------------------------------------------------------- /redis/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | all: 4 | $(MAKE) -C 7.4.1 5 | $(MAKE) -C 7.2.4 6 | $(MAKE) -C 7.0.11 7 | $(MAKE) -C 6.2.1 8 | $(MAKE) -C 5.0.14 9 | $(MAKE) -C 4.0.6 10 | $(MAKE) -C 3.2.11 11 | $(MAKE) -C 2.8.24 12 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/AuxField.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | /** 16 | *

An auxiliary field contains a key value pair that holds metadata about the RDB file. 17 | * 18 | *

Introduced in RDB version 7. 19 | * 20 | * @author John Whitbeck 21 | */ 22 | public final class AuxField implements Entry { 23 | 24 | private final byte[] key; 25 | private final byte[] value; 26 | 27 | AuxField(byte[] key, byte[] value) { 28 | this.key = key; 29 | this.value = value; 30 | } 31 | 32 | @Override 33 | public EntryType getType() { 34 | return EntryType.AUX_FIELD; 35 | } 36 | 37 | /** 38 | * Returns the key. 39 | * 40 | * @return key 41 | */ 42 | public byte[] getKey() { 43 | return key; 44 | } 45 | 46 | /** 47 | * Returns the value. 48 | * 49 | * @return value 50 | */ 51 | public byte[] getValue() { 52 | return value; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return String.format("%s (k: %s, v: %s)", 58 | EntryType.AUX_FIELD, 59 | StringUtils.getPrintableString(key), 60 | StringUtils.getPrintableString(value)); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/DoubleBytes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.nio.charset.Charset; 16 | 17 | final class DoubleBytes { 18 | 19 | private static final Charset ASCII = Charset.forName("ASCII"); 20 | 21 | static final byte[] POSITIVE_INFINITY = String.valueOf(Double.POSITIVE_INFINITY).getBytes(ASCII); 22 | static final byte[] NEGATIVE_INFINITY = String.valueOf(Double.NEGATIVE_INFINITY).getBytes(ASCII); 23 | static final byte[] NaN = String.valueOf(Double.NaN).getBytes(ASCII); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/Entry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | public interface Entry { 16 | 17 | /** 18 | * Returns one of EOF, SELECT_DB, KEY_VALUE_PAIR, RESIZE_DB, or AUX_FIELD. 19 | * 20 | * @return the entry type. 21 | */ 22 | EntryType getType(); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/EntryType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | /** 16 | * This enum holds the different types of entries that the {@link RdbParser} can read from a RDB file. 17 | * 18 | * @author John Whitbeck 19 | */ 20 | public enum EntryType { 21 | 22 | /** 23 | * Denotes an end-of-file entry with a checksum. These entries are marked by a 0xff byte in the 24 | * RDB file. 25 | * 26 | * @see Eof 27 | */ 28 | EOF, 29 | 30 | /** 31 | * Denotes a DB selection entry. These entries are marked by a 0xfe byte in the RDB file. 32 | * 33 | * @see SelectDb 34 | */ 35 | SELECT_DB, 36 | 37 | /** 38 | * Denotes a key/value pair entry that may optionally have an expire time, an LFU frequency, or an 39 | * LRU idle time. In the RDB file, these entries are marked by a 0xfd byte (expire time in 40 | * seconds), a 0xfc byte (expire time in milliseconds), a 0xf9 byte (LFU frequency), a 0xf8 byte 41 | * (LRU idle time), or no marker (no expire time). 42 | * 43 | * @see KeyValuePair 44 | */ 45 | KEY_VALUE_PAIR, 46 | 47 | /** 48 | * Denotes an entry containing the database hash table size and the expire time hash table 49 | * size. These entries are marked by a 0xfb byte in the RDB file. 50 | * 51 | * @see ResizeDb 52 | */ 53 | RESIZE_DB, 54 | 55 | /** 56 | * Denotes an auxiliary field for storing a key/value pair containing metadata about the RDB 57 | * file. These entries are marked by a 0xfa byte in the RDB file. 58 | * 59 | * @see AuxField 60 | */ 61 | AUX_FIELD 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/Eof.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | /** 16 | *

End-of-file entry. This is always the last entry in the file and, as of RDB version 6, 17 | * contains an 8 byte checksum of the file. 18 | * 19 | * @author John Whitbeck 20 | */ 21 | public final class Eof implements Entry { 22 | 23 | private final byte[] checksum; 24 | 25 | Eof(byte[] checksum) { 26 | this.checksum = checksum; 27 | } 28 | 29 | @Override 30 | public EntryType getType() { 31 | return EntryType.EOF; 32 | } 33 | 34 | /** 35 | * Returns the 8-byte checksum of the rdb file. These 8 bytes are all zero if the version of RDB 36 | * file is 4 or older. 37 | * 38 | * @return the 8-byte checksum of the rdb file. 39 | */ 40 | public byte[] getChecksum() { 41 | return checksum; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | StringBuilder sb = new StringBuilder(); 47 | sb.append(EntryType.EOF); 48 | sb.append(" ("); 49 | for (byte b : checksum) { 50 | sb.append(String.format("%02x", (int)b & 0xff)); 51 | } 52 | sb.append(")"); 53 | return sb.toString(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/IntSet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.nio.charset.Charset; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | final class IntSet extends LazyList { 20 | 21 | private static final Charset ASCII = Charset.forName("ASCII"); 22 | 23 | private final byte[] envelope; 24 | 25 | IntSet(byte[] envelope) { 26 | this.envelope = envelope; 27 | } 28 | 29 | private int readIntAt(int pos) { 30 | return ((int)envelope[pos++] & 0xff) << 0 31 | | ((int)envelope[pos++] & 0xff) << 8 32 | | ((int)envelope[pos++] & 0xff) << 16 33 | | ((int)envelope[pos++] & 0xff) << 24; 34 | } 35 | 36 | private int getEncoding() { 37 | // Encoding can take three values: 2, 4, or 8, stored as a little-endian 32 bit integer. 38 | return readIntAt(0); 39 | } 40 | 41 | private int getNumInts() { 42 | // Number of ints is stored as a little-endian 32 bit integer, stored right after the encoding. 43 | return readIntAt(4); 44 | } 45 | 46 | private List read16BitInts(int num) { 47 | List ints = new ArrayList(num); 48 | int pos = 8; // skip the encoding and num ints 49 | for (int i = 0; i < num; ++i) { 50 | long val = ((long)envelope[pos++] & 0xff) << 0 51 | | (long)envelope[pos++] << 8; 52 | ints.add(String.valueOf(val).getBytes(ASCII)); 53 | } 54 | return ints; 55 | } 56 | 57 | private List read32BitInts(int num) { 58 | List ints = new ArrayList(num); 59 | int pos = 8; // skip the encoding and num ints 60 | for (int i = 0; i < num; ++i) { 61 | long val = ((long)envelope[pos++] & 0xff) << 0 62 | | ((long)envelope[pos++] & 0xff) << 8 63 | | ((long)envelope[pos++] & 0xff) << 16 64 | | ((long)envelope[pos++]) << 24; 65 | ints.add(String.valueOf(val).getBytes(ASCII)); 66 | } 67 | return ints; 68 | } 69 | 70 | private List read64BitInts(int num) { 71 | List ints = new ArrayList(num); 72 | int pos = 8; // skip the encoding and num ints 73 | for (int i = 0; i < num; ++i) { 74 | long val = ((long)envelope[pos++] & 0xff) << 0 75 | | ((long)envelope[pos++] & 0xff) << 8 76 | | ((long)envelope[pos++] & 0xff) << 16 77 | | ((long)envelope[pos++] & 0xff) << 24 78 | | ((long)envelope[pos++] & 0xff) << 32 79 | | ((long)envelope[pos++] & 0xff) << 40 80 | | ((long)envelope[pos++] & 0xff) << 48 81 | | (long)envelope[pos++] << 56; 82 | ints.add(String.valueOf(val).getBytes(ASCII)); 83 | } 84 | return ints; 85 | } 86 | 87 | @Override 88 | protected List realize() { 89 | int encoding = getEncoding(); 90 | int num = getNumInts(); 91 | switch (encoding) { 92 | case 2: 93 | return read16BitInts(num); 94 | case 4: 95 | return read32BitInts(num); 96 | case 8: 97 | return read64BitInts(num); 98 | default: 99 | throw new IllegalStateException("Unknown intset encoding"); 100 | } 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/KeyValuePair.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | *

Key/value pair entries contain all the data associated with a given key. 19 | * 20 | *

This data includes: 21 | * 22 | *

30 | * 31 | * @author John Whitbeck 32 | */ 33 | public final class KeyValuePair implements Entry { 34 | 35 | byte[] key; 36 | ValueType valueType; 37 | List values; 38 | byte[] expireTime; 39 | Long idle; 40 | Integer freq; 41 | Long minHashExpireTime; 42 | 43 | /** 44 | * Returns the key associated with this key/value pair. 45 | * 46 | * @return the key 47 | */ 48 | public byte[] getKey() { 49 | return key; 50 | } 51 | 52 | /** 53 | * Returns the value type encoding. 54 | * 55 | * @return the value type encoding. 56 | */ 57 | public ValueType getValueType() { 58 | return valueType; 59 | } 60 | 61 | @Override 62 | public EntryType getType() { 63 | return EntryType.KEY_VALUE_PAIR; 64 | } 65 | 66 | /** 67 | * Returns the list of values (as byte-arrays) associated with this key/value pair. 68 | * 69 | *

The values in this list depend on the value type. 70 | * 71 | *

81 | * 82 | * @return the list of values. 83 | */ 84 | public List getValues() { 85 | return values; 86 | } 87 | 88 | /** 89 | * Returns the expire time in milliseconds. If the initial expire time was set in seconds in 90 | * redis, the expire time is converted to milliseconds. Returns null if no expire time is set. 91 | * 92 | * @return the expire time in milliseconds. 93 | */ 94 | public Long getExpireTime() { 95 | if (expireTime == null) { 96 | return null; 97 | } 98 | switch (expireTime.length) { 99 | case 4: 100 | return parseExpireTime4Bytes(); 101 | case 8: 102 | return parseExpireTime8Bytes(); 103 | default: 104 | throw new IllegalStateException("Invalid number of expire time bytes"); 105 | } 106 | } 107 | 108 | private long parseExpireTime4Bytes() { 109 | return 1000L * ( ((long)expireTime[3] & 0xff) << 24 110 | | ((long)expireTime[2] & 0xff) << 16 111 | | ((long)expireTime[1] & 0xff) << 8 112 | | ((long)expireTime[0] & 0xff) << 0); 113 | } 114 | 115 | private long parseExpireTime8Bytes() { 116 | return ((long)expireTime[7] & 0xff) << 56 117 | | ((long)expireTime[6] & 0xff) << 48 118 | | ((long)expireTime[5] & 0xff) << 40 119 | | ((long)expireTime[4] & 0xff) << 32 120 | | ((long)expireTime[3] & 0xff) << 24 121 | | ((long)expireTime[2] & 0xff) << 16 122 | | ((long)expireTime[1] & 0xff) << 8 123 | | ((long)expireTime[0] & 0xff) << 0; 124 | } 125 | 126 | public Long getMinHashExpireTime() { 127 | switch (valueType) { 128 | case HASHMAP_WITH_METADATA: 129 | case HASHMAP_WITH_METADATA_PRE_GA: 130 | case HASHMAP_AS_LISTPACK_EX: 131 | case HASHMAP_AS_LISTPACK_EX_PRE_GA: 132 | return minHashExpireTime; 133 | default: 134 | return null; 135 | } 136 | } 137 | 138 | /** 139 | * Returns the LFU frequency (logarithmic with a 0-255 range) , or null if not set. 140 | * 141 | * @return the LFU frequency 142 | */ 143 | public Integer getFreq() { 144 | return freq; 145 | } 146 | 147 | /** 148 | * Returns the LRU frequency (in seconds), or null if not set. 149 | * 150 | * @return the LRU idle time 151 | */ 152 | public Long getIdle() { 153 | return idle; 154 | } 155 | 156 | @Override 157 | public String toString() { 158 | StringBuilder sb = new StringBuilder(); 159 | sb.append(EntryType.KEY_VALUE_PAIR); 160 | sb.append(" (key: "); 161 | sb.append(StringUtils.getPrintableString(key)); 162 | if (expireTime != null) { 163 | sb.append(", expire time: "); 164 | sb.append(getExpireTime()); 165 | } 166 | sb.append(", "); 167 | int len = getValues().size(); 168 | sb.append(len); 169 | if (len == 1) { 170 | sb.append(" value)"); 171 | } else { 172 | sb.append(" values)"); 173 | } 174 | if (minHashExpireTime != null) { 175 | sb.append(", min hash expire time: "); 176 | sb.append(minHashExpireTime); 177 | } 178 | return sb.toString(); 179 | } 180 | 181 | } 182 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/LazyList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.util.AbstractSequentialList; 16 | import java.util.List; 17 | import java.util.ListIterator; 18 | 19 | abstract class LazyList extends AbstractSequentialList { 20 | 21 | private List list = null; 22 | 23 | protected abstract List realize(); 24 | 25 | @Override 26 | public ListIterator listIterator(int index) { 27 | if (list == null){ 28 | list = realize(); 29 | } 30 | return list.listIterator(index); 31 | } 32 | 33 | @Override 34 | public int size() { 35 | if (list == null){ 36 | list = realize(); 37 | } 38 | return list.size(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/ListpackList.java: -------------------------------------------------------------------------------- 1 | package net.whitbeck.rdbparser; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Arrays; 6 | import java.nio.charset.Charset; 7 | 8 | class ListpackList extends LazyList { 9 | private static final Charset ASCII = Charset.forName("ASCII"); 10 | 11 | // Taken from 12 | // https://github.com/redis/redis/blob/7.0.11/src/listpack.c#L55-L95C4 13 | private static final int LP_ENCODING_7BIT_UINT = 0; 14 | private static final int LP_ENCODING_7BIT_UINT_MASK = 0x80; 15 | private static final int LP_ENCODING_6BIT_STR = 0x80; 16 | private static final int LP_ENCODING_6BIT_STR_MASK = 0xC0; 17 | private static final int LP_ENCODING_13BIT_INT = 0xC0; 18 | private static final int LP_ENCODING_13BIT_INT_MASK = 0xE0; 19 | private static final int LP_ENCODING_12BIT_STR = 0xE0; 20 | private static final int LP_ENCODING_12BIT_STR_MASK = 0xF0; 21 | 22 | // Sub encodings 23 | private static final int LP_ENCODING_16BIT_INT = 0xF1; 24 | private static final int LP_ENCODING_16BIT_INT_MASK = 0xFF; 25 | private static final int LP_ENCODING_24BIT_INT = 0xF2; 26 | private static final int LP_ENCODING_24BIT_INT_MASK = 0xFF; 27 | private static final int LP_ENCODING_32BIT_INT = 0xF3; 28 | private static final int LP_ENCODING_32BIT_INT_MASK = 0xFF; 29 | private static final int LP_ENCODING_64BIT_INT = 0xF4; 30 | private static final int LP_ENCODING_64BIT_INT_MASK = 0xFF; 31 | private static final int LP_ENCODING_32BIT_STR = 0xF0; 32 | private static final int LP_ENCODING_32BIT_STR_MASK = 0xFF; 33 | 34 | private final byte[] envelope; 35 | 36 | ListpackList(byte[] envelope) { 37 | this.envelope = envelope; 38 | } 39 | 40 | private class ListpackParser { 41 | private int pos = 0; 42 | private List list = new ArrayList(); 43 | 44 | private void decodeElement() { 45 | int b = envelope[pos++] & 0xff; 46 | 47 | // Handle the string cases first. 48 | int strLen = 0; 49 | 50 | if ((b & LP_ENCODING_6BIT_STR_MASK) == LP_ENCODING_6BIT_STR) { 51 | // 10|xxxxxx with x being the str length. 52 | strLen = b & ~LP_ENCODING_6BIT_STR_MASK; 53 | } else if ((b & LP_ENCODING_12BIT_STR_MASK) == LP_ENCODING_12BIT_STR) { 54 | // 1110|xxxxx yyyyyyyy str len up to 4095. 55 | strLen = ((int)envelope[pos++] & 0xff) 56 | | (b & 0xff & ~LP_ENCODING_12BIT_STR_MASK) << 8; 57 | } else if ((b & LP_ENCODING_32BIT_STR_MASK) == LP_ENCODING_32BIT_STR) { 58 | // 1100|0000 subencoding. 59 | strLen = ((int)envelope[pos++] & 0xff) << 0 60 | | ((int)envelope[pos++] & 0xff) << 8 61 | | ((int)envelope[pos++] & 0xff) << 16 62 | | (int)envelope[pos++] << 24; 63 | } 64 | 65 | if (strLen > 0) { 66 | pos += strLen; 67 | list.add(Arrays.copyOfRange(envelope, pos - strLen, pos)); 68 | pos += getLenBytes(strLen); 69 | return; 70 | } 71 | 72 | // Handle the ints. 73 | long val, negStart, negMax; 74 | 75 | if ((b & LP_ENCODING_7BIT_UINT_MASK) == LP_ENCODING_7BIT_UINT) { 76 | // Small number encoded in a single byte. 77 | list.add(String.valueOf(b & ~LP_ENCODING_7BIT_UINT_MASK).getBytes(ASCII)); 78 | pos++; 79 | // Return immediately since 7-bit ints are never negative. 80 | return; 81 | } else if ((b & LP_ENCODING_13BIT_INT_MASK) == LP_ENCODING_13BIT_INT) { // 110|xxxxxx 82 | // yyyyyyyy 83 | val = (b & 0xff & ~LP_ENCODING_13BIT_INT_MASK) << 8 | envelope[pos++] & 0xff; 84 | negStart = 1 << 12; 85 | negMax = (1 << 13) - 1; 86 | } else if ((b & LP_ENCODING_16BIT_INT_MASK) == LP_ENCODING_16BIT_INT) { // 1111|0001 87 | val = ((long) envelope[pos++] & 0xff) | ((long) envelope[pos++] & 0xff) << 8; 88 | negStart = 1 << 15; 89 | negMax = (1 << 16) - 1; 90 | } else if ((b & LP_ENCODING_24BIT_INT_MASK) == LP_ENCODING_24BIT_INT) { // 1100|0010 91 | val = ((long) envelope[pos++] & 0xff) | ((long) envelope[pos++] & 0xff) << 8 92 | | ((long) envelope[pos++] & 0xff) << 16; 93 | negStart = 1L << 23; 94 | negMax = (1L << 24) - 1; 95 | } else if ((b & LP_ENCODING_32BIT_INT_MASK) == LP_ENCODING_32BIT_INT) { // 1100|0011 96 | val = ((long) envelope[pos++] & 0xff) | ((long) envelope[pos++] & 0xff) << 8 97 | | ((long) envelope[pos++] & 0xff) << 16 98 | | ((long) envelope[pos++] & 0xff) << 24; 99 | negStart = 1L << 31; 100 | negMax = (1L << 32) - 1; 101 | } else if ((b & LP_ENCODING_64BIT_INT_MASK) == LP_ENCODING_64BIT_INT) { // 1100|0100 102 | val = ((long) envelope[pos++] & 0xff) | ((long) envelope[pos++] & 0xff) << 8 103 | | ((long) envelope[pos++] & 0xff) << 16 104 | | ((long) envelope[pos++] & 0xff) << 24 105 | | ((long) envelope[pos++] & 0xff) << 32 106 | | ((long) envelope[pos++] & 0xff) << 40 107 | | ((long) envelope[pos++] & 0xff) << 48 108 | | ((long) envelope[pos++] & 0xff) << 56; 109 | // Since a long is 64 bits, no negative correction is needed. 110 | list.add(String.valueOf(val).getBytes(ASCII)); 111 | pos++; 112 | return; 113 | } else { 114 | throw new RuntimeException("Invalid listpack envelope encoding"); 115 | } 116 | 117 | // Convert to two's complement if value is negative. 118 | if (val >= negStart) { 119 | 120 | long diff = negMax - val; 121 | val = diff; 122 | val = -val - 1; 123 | } 124 | // Ints always have a entity size of one byte. 125 | pos++; 126 | list.add(String.valueOf(val).getBytes(ASCII)); 127 | } 128 | 129 | private int getLenBytes(int len) { 130 | if (len < 128) { 131 | return 1; 132 | } else if (len < 16384) { 133 | return 2; 134 | } else if (len < 2097152) { 135 | return 3; 136 | } else if (len < 268435456) { 137 | return 4; 138 | } else { 139 | return 5; 140 | } 141 | } 142 | } 143 | 144 | @Override 145 | protected List realize() { 146 | // The structure of the listpack is: 147 | // ... 148 | // Where each element is of the structure: 149 | // . 150 | // Reference: https://github.com/antirez/listpack/blob/master/listpack.md 151 | 152 | ListpackParser listpackParser = new ListpackParser(); 153 | // Skip 32-bit integer for the total number of bytes in listpack. 154 | listpackParser.pos += 4; 155 | int numElements = ((int) envelope[listpackParser.pos++] & 0xff) << 0 156 | | ((int) envelope[listpackParser.pos++] & 0xff) << 8; 157 | 158 | for (int i = 0; i < numElements; i++) { 159 | listpackParser.decodeElement(); 160 | } 161 | if ((envelope[listpackParser.pos] & 0xff) != 0xff) { 162 | throw new IllegalStateException("Listpack did not end with 0xff byte."); 163 | } 164 | return listpackParser.list; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/Lzf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | // adapted from https://github.com/ganghuawang/java-redis-rdb 16 | final class Lzf { 17 | 18 | // The maximum number of literals in a chunk (32). 19 | private static int MAX_LITERAL = 32; 20 | 21 | static void expand(byte[] src, byte[] dest) { 22 | int srcPos = 0; 23 | int destPos = 0; 24 | do { 25 | int ctrl = src[srcPos++] & 0xff; 26 | if (ctrl < MAX_LITERAL) { 27 | // literal run of length = ctrl + 1, 28 | ctrl++; 29 | // copy to output and move forward this many bytes 30 | System.arraycopy(src, srcPos, dest, destPos, ctrl); 31 | destPos += ctrl; 32 | srcPos += ctrl; 33 | } else { 34 | /* back reference 35 | the highest 3 bits are the match length */ 36 | int len = ctrl >> 5; 37 | // if the length is maxed, add the next byte to the length 38 | if (len == 7) { 39 | len += src[srcPos++] & 0xff; 40 | } 41 | /* minimum back-reference is 3 bytes, 42 | so 2 was subtracted before storing size */ 43 | len += 2; 44 | 45 | /* ctrl is now the offset for a back-reference... 46 | the logical AND operation removes the length bits */ 47 | ctrl = -((ctrl & 0x1f) << 8) - 1; 48 | 49 | // the next byte augments/increases the offset 50 | ctrl -= src[srcPos++] & 0xff; 51 | 52 | /* copy the back-reference bytes from the given 53 | location in output to current position */ 54 | ctrl += destPos; 55 | for (int i = 0; i < len; i++) { 56 | dest[destPos++] = dest[ctrl++]; 57 | } 58 | } 59 | } while (destPos < dest.length); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/QuickList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | final class QuickList extends LazyList { 19 | 20 | private final List ziplists; 21 | 22 | QuickList(List ziplists) { 23 | this.ziplists = ziplists; 24 | } 25 | 26 | @Override 27 | protected List realize() { 28 | List list = new ArrayList(); 29 | for (byte[] envelope : ziplists) { 30 | list.addAll(new ZipList(envelope).realize()); 31 | } 32 | return list; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/QuickList2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | final class QuickList2 extends LazyList { 19 | private final List listpacks; 20 | 21 | QuickList2(List listpacks) { 22 | this.listpacks = listpacks; 23 | } 24 | 25 | @Override 26 | protected List realize() { 27 | List list = new ArrayList(); 28 | for (byte[] listpack : listpacks) { 29 | list.addAll(new ListpackList(listpack).realize()); 30 | } 31 | return list; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/RdbParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.io.File; 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.nio.ByteBuffer; 19 | import java.nio.ByteOrder; 20 | import java.nio.channels.Channels; 21 | import java.nio.channels.FileChannel; 22 | import java.nio.channels.ReadableByteChannel; 23 | import java.nio.charset.Charset; 24 | import java.nio.file.Path; 25 | import java.nio.file.StandardOpenOption; 26 | import java.util.ArrayList; 27 | import java.util.Arrays; 28 | import java.util.List; 29 | 30 | /** 31 | *

Reads entries from a Redis RDB file, one at a time. 32 | * 33 | * @author John Whitbeck 34 | */ 35 | public final class RdbParser implements AutoCloseable { 36 | 37 | private static final Charset ASCII = Charset.forName("ASCII"); 38 | 39 | private static final int EOF = 0xff; 40 | private static final int SELECTDB = 0xfe; 41 | private static final int EXPIRETIME = 0xfd; 42 | private static final int EXPIRETIME_MS = 0xfc; 43 | private static final int RESIZEDB = 0xfb; 44 | private static final int AUX = 0xfa; 45 | private static final int FREQ = 0xf9; 46 | private static final int IDLE = 0xf8; 47 | private static final int MODULE_AUX = 0xf7; 48 | private static final int FUNCTION_PRE_GA = 0xf6; 49 | private static final int FUNCTION2 = 0xf5; 50 | private static final int SLOT_INFO = 0xf4; 51 | 52 | private static final int BUFFER_SIZE = 8 * 1024; 53 | 54 | private final ReadableByteChannel ch; 55 | private final ByteBuffer buf = ByteBuffer.allocateDirect(BUFFER_SIZE); 56 | 57 | /* Parsing state */ 58 | private int version; 59 | private long bytesBuffered = 0; 60 | private boolean isInitialized = false; 61 | private KeyValuePair nextEntry = null; 62 | private boolean hasNext = false; 63 | 64 | public RdbParser(ReadableByteChannel ch) { 65 | this.ch = ch; 66 | } 67 | 68 | public RdbParser(Path path) throws IOException { 69 | this.ch = FileChannel.open(path, StandardOpenOption.READ); 70 | } 71 | 72 | public RdbParser(File file) throws IOException { 73 | this(file.toPath()); 74 | } 75 | 76 | public RdbParser(InputStream inputStream) throws IOException { 77 | this(Channels.newChannel(inputStream)); 78 | } 79 | 80 | public RdbParser(String filename) throws IOException { 81 | this(new File(filename)); 82 | } 83 | 84 | /** 85 | * Returns the version of the RDB file being parsed. 86 | * 87 | * @return the RDB file version 88 | */ 89 | public int getRdbVersion() { 90 | return version; 91 | } 92 | 93 | private void fillBuffer() throws IOException { 94 | buf.clear(); 95 | long n = ch.read(buf); 96 | if (n == -1) { 97 | throw new IOException("Attempting to read past channel end-of-stream."); 98 | } 99 | bytesBuffered += n; 100 | buf.flip(); 101 | } 102 | 103 | private int readByte() throws IOException { 104 | if (!buf.hasRemaining()) { 105 | fillBuffer(); 106 | } 107 | return buf.get() & 0xff; 108 | } 109 | 110 | private int readSignedByte() throws IOException { 111 | if (!buf.hasRemaining()) { 112 | fillBuffer(); 113 | } 114 | return buf.get(); 115 | } 116 | 117 | private byte[] readBytes(int numBytes) throws IOException { 118 | int rem = numBytes; 119 | int pos = 0; 120 | byte[] bs = new byte[numBytes]; 121 | while (rem > 0) { 122 | int avail = buf.remaining(); 123 | if (avail >= rem) { 124 | buf.get(bs, pos, rem); 125 | pos += rem; 126 | rem = 0; 127 | } else { 128 | buf.get(bs, pos, avail); 129 | pos += avail; 130 | rem -= avail; 131 | fillBuffer(); 132 | } 133 | } 134 | return bs; 135 | } 136 | 137 | private long readExpirationMillis() throws IOException { 138 | byte[] expireTime = readBytes(8); 139 | return ((long)expireTime[7] & 0xff) << 56 140 | | ((long)expireTime[6] & 0xff) << 48 141 | | ((long)expireTime[5] & 0xff) << 40 142 | | ((long)expireTime[4] & 0xff) << 32 143 | | ((long)expireTime[3] & 0xff) << 24 144 | | ((long)expireTime[2] & 0xff) << 16 145 | | ((long)expireTime[1] & 0xff) << 8 146 | | ((long)expireTime[0] & 0xff) << 0; 147 | } 148 | 149 | private String readMagicNumber() throws IOException { 150 | return new String(readBytes(5), ASCII); 151 | } 152 | 153 | private int readVersion() throws IOException { 154 | return Integer.parseInt(new String(readBytes(4), ASCII)); 155 | } 156 | 157 | private void init() throws IOException { 158 | fillBuffer(); 159 | if (!readMagicNumber().equals("REDIS")) { 160 | throw new IllegalStateException("Not a valid redis RDB file"); 161 | } 162 | version = readVersion(); 163 | if (version < 1 || version > 12) { 164 | throw new IllegalStateException("Unknown version"); 165 | } 166 | nextEntry = new KeyValuePair(); 167 | hasNext = true; 168 | isInitialized = true; 169 | } 170 | 171 | /** 172 | *

Returns the number of bytes parsed from the underlying file or stream by successive calls of 173 | * the {@link #readNext} method. 174 | * 175 | *

As RdbParser uses a buffer internally, the returned value will be slightly smaller than the 176 | * total number of bytes buffered from the underlying file or stream. 177 | * 178 | * @return the number of bytes parsed so far. 179 | */ 180 | public long bytesParsed() { 181 | return bytesBuffered - buf.remaining(); 182 | } 183 | 184 | /** 185 | * Returns the next Entry from the underlying file or stream. 186 | * 187 | * @return the next entry 188 | * 189 | * @throws IOException if there is an error reading from the underlying channel. 190 | */ 191 | public Entry readNext() throws IOException { 192 | while (true) { 193 | if (!hasNext) { 194 | if (!isInitialized) { 195 | init(); 196 | continue; 197 | } else { // EOF reached 198 | return null; 199 | } 200 | } 201 | int valueType = readByte(); 202 | switch (valueType) { 203 | case EOF: 204 | return readEof(); 205 | case SELECTDB: 206 | return readSelectDb(); 207 | case RESIZEDB: 208 | return readResizeDb(); 209 | case AUX: 210 | return readAuxField(); 211 | case EXPIRETIME: 212 | readExpireTime(); 213 | continue; 214 | case EXPIRETIME_MS: 215 | readExpireTimeMillis(); 216 | continue; 217 | case FREQ: 218 | readFreq(); 219 | continue; 220 | case IDLE: 221 | readIdle(); 222 | continue; 223 | case MODULE_AUX: 224 | throw new UnsupportedOperationException("Redis modules are not supported"); 225 | case FUNCTION_PRE_GA: 226 | case FUNCTION2: 227 | throw new UnsupportedOperationException("Redis functions are not supported"); 228 | case SLOT_INFO: 229 | throw new UnsupportedOperationException("Redis cluster is not supported"); 230 | default: 231 | readEntry(valueType); 232 | KeyValuePair entry = nextEntry; 233 | nextEntry = new KeyValuePair(); 234 | return entry; 235 | } 236 | } 237 | } 238 | 239 | private byte[] readChecksum() throws IOException { 240 | return readBytes(8); 241 | } 242 | 243 | private byte[] getEmptyChecksum() { 244 | return new byte[8]; 245 | } 246 | 247 | private Eof readEof() throws IOException { 248 | byte[] checksum = version >= 5 ? readChecksum() : getEmptyChecksum(); 249 | hasNext = false; 250 | return new Eof(checksum); 251 | } 252 | 253 | private SelectDb readSelectDb() throws IOException { 254 | return new SelectDb(readLength()); 255 | } 256 | 257 | private ResizeDb readResizeDb() throws IOException { 258 | return new ResizeDb(readLength(), readLength()); 259 | } 260 | 261 | private AuxField readAuxField() throws IOException { 262 | return new AuxField(readStringEncoded(), readStringEncoded()); 263 | } 264 | 265 | private void readFreq() throws IOException { 266 | nextEntry.freq = readByte(); 267 | } 268 | 269 | private void readIdle() throws IOException { 270 | nextEntry.idle = readLength(); 271 | } 272 | 273 | private long readLength() throws IOException { 274 | int firstByte = readByte(); 275 | // The first two bits determine the encoding. 276 | int flag = (firstByte & 0xc0) >> 6; 277 | if (flag == 0) { // 00|XXXXXX: len is the last 6 bits of this byte. 278 | return firstByte & 0x3f; 279 | } else if (flag == 1) { // 01|XXXXXX: len is encoded on the next 14 bits. 280 | return (((long)firstByte & 0x3f) << 8) | ((long)readByte() & 0xff); 281 | } else if (firstByte == 0x80) { 282 | // 10|000000: len is a 32-bit integer encoded on the next 4 bytes. 283 | byte[] bs = readBytes(4); 284 | return ((long)bs[0] & 0xff) << 24 285 | | ((long)bs[1] & 0xff) << 16 286 | | ((long)bs[2] & 0xff) << 8 287 | | ((long)bs[3] & 0xff) << 0; 288 | } else if (firstByte == 0x81) { 289 | // 10|000001: len is a 64-bit integer encoded on the next 8 bytes. 290 | byte[] bs = readBytes(8); 291 | return ((long)bs[0] & 0xff) << 56 292 | | ((long)bs[1] & 0xff) << 48 293 | | ((long)bs[2] & 0xff) << 40 294 | | ((long)bs[3] & 0xff) << 32 295 | | ((long)bs[4] & 0xff) << 24 296 | | ((long)bs[5] & 0xff) << 16 297 | | ((long)bs[6] & 0xff) << 8 298 | | ((long)bs[7] & 0xff) << 0; 299 | } else { 300 | // 11|XXXXXX: special encoding. 301 | throw new IllegalStateException("Expected a length, but got a special string encoding."); 302 | } 303 | } 304 | 305 | private byte[] readStringEncoded() throws IOException { 306 | int firstByte = readByte(); 307 | // the first two bits determine the encoding 308 | int flag = (firstByte & 0xc0) >> 6; 309 | int len; 310 | switch (flag) { 311 | case 0: // length is read from the lower 6 bits 312 | len = firstByte & 0x3f; 313 | return readBytes(len); 314 | case 1: // one additional byte is read for a 14 bit encoding 315 | len = ((firstByte & 0x3f) << 8) | (readByte() & 0xff); 316 | return readBytes(len); 317 | case 2: // read next four bytes as unsigned big-endian 318 | byte[] bs = readBytes(4); 319 | len = ((int)bs[0] & 0xff) << 24 320 | | ((int)bs[1] & 0xff) << 16 321 | | ((int)bs[2] & 0xff) << 8 322 | | ((int)bs[3] & 0xff) << 0; 323 | if (len < 0) { 324 | throw new IllegalStateException("Strings longer than " + Integer.MAX_VALUE 325 | + "bytes are not supported."); 326 | } 327 | return readBytes(len); 328 | case 3: 329 | return readSpecialStringEncoded(firstByte & 0x3f); 330 | default: // never reached 331 | return null; 332 | } 333 | } 334 | 335 | private byte[] readInteger8Bits() throws IOException { 336 | return String.valueOf(readSignedByte()).getBytes(ASCII); 337 | } 338 | 339 | private byte[] readInteger16Bits() throws IOException { 340 | long val = ((long)readByte() & 0xff) << 0 341 | | (long)readSignedByte() << 8; // Don't apply 0xff mask to preserve sign. 342 | return String.valueOf(val).getBytes(ASCII); 343 | } 344 | 345 | private byte[] readInteger32Bits() throws IOException { 346 | byte[] bs = readBytes(4); 347 | long val = (long)bs[3] << 24 // Don't apply 0xff mask to preserve sign. 348 | | ((long)bs[2] & 0xff) << 16 349 | | ((long)bs[1] & 0xff) << 8 350 | | ((long)bs[0] & 0xff) << 0; 351 | return String.valueOf(val).getBytes(ASCII); 352 | } 353 | 354 | private byte[] readLzfString() throws IOException { 355 | int clen = (int)readLength(); 356 | int ulen = (int)readLength(); 357 | byte[] src = readBytes(clen); 358 | byte[] dest = new byte[ulen]; 359 | Lzf.expand(src, dest); 360 | return dest; 361 | } 362 | 363 | private byte[] readDoubleString() throws IOException { 364 | int len = readByte(); 365 | switch (len) { 366 | case 0xff: 367 | return DoubleBytes.NEGATIVE_INFINITY; 368 | case 0xfe: 369 | return DoubleBytes.POSITIVE_INFINITY; 370 | case 0xfd: 371 | return DoubleBytes.NaN; 372 | default: 373 | return readBytes(len); 374 | } 375 | } 376 | 377 | private byte[] readSpecialStringEncoded(int type) throws IOException { 378 | switch (type) { 379 | case 0: 380 | return readInteger8Bits(); 381 | case 1: 382 | return readInteger16Bits(); 383 | case 2: 384 | return readInteger32Bits(); 385 | case 3: 386 | return readLzfString(); 387 | default: 388 | throw new IllegalStateException("Unknown special encoding: " + type); 389 | } 390 | } 391 | 392 | private void readExpireTime() throws IOException { 393 | nextEntry.expireTime = readBytes(4); 394 | } 395 | 396 | private void readExpireTimeMillis() throws IOException { 397 | nextEntry.expireTime = readBytes(8); 398 | } 399 | 400 | private void readEntry(int valueType) throws IOException { 401 | nextEntry.key = readStringEncoded(); 402 | switch (valueType) { 403 | case 0: 404 | readValue(); 405 | break; 406 | case 1: 407 | readList(); 408 | break; 409 | case 2: 410 | readSet(); 411 | break; 412 | case 3: 413 | readSortedSet(); 414 | break; 415 | case 4: 416 | readHash(); 417 | break; 418 | case 5: 419 | readSortedSet2(); 420 | break; 421 | case 6: // Modules v1 422 | case 7: // Modules v2 423 | throw new UnsupportedOperationException("Redis modules are not supported"); 424 | case 9: 425 | readZipMap(); 426 | break; 427 | case 10: 428 | readZipList(); 429 | break; 430 | case 11: 431 | readIntSet(); 432 | break; 433 | case 12: 434 | readSortedSetAsZipList(); 435 | break; 436 | case 13: 437 | readHashmapAsZipList(); 438 | break; 439 | case 14: 440 | readQuickList(); 441 | break; 442 | case 15: // Stream ListPacks 443 | case 19: // Stream ListPacks_2 444 | case 21: // Stream ListPacks_3 445 | throw new UnsupportedOperationException("Redis streams are not supported"); 446 | case 16: 447 | readHashListPack(); 448 | break; 449 | case 17: 450 | readZSetListPack(); 451 | break; 452 | case 18: 453 | readQuickList2(); 454 | break; 455 | case 20: 456 | readSetListPack(); 457 | break; 458 | case 22: 459 | case 24: 460 | readHashMetadata(valueType == 24); 461 | break; 462 | case 23: 463 | case 25: 464 | readHashListPackEx(valueType == 25); 465 | break; 466 | default: 467 | throw new UnsupportedOperationException("Unknown value type: " + valueType); 468 | } 469 | } 470 | 471 | private void readZSetListPack() throws IOException { 472 | nextEntry.valueType = ValueType.SORTED_SET_AS_LISTPACK; 473 | nextEntry.values = new SortedSetAsListpack(readStringEncoded()); 474 | } 475 | 476 | private void readSetListPack() throws IOException { 477 | nextEntry.valueType = ValueType.SET_AS_LISTPACK; 478 | nextEntry.values = new ListpackList(readStringEncoded()); 479 | } 480 | 481 | private void readValue() throws IOException { 482 | nextEntry.valueType = ValueType.VALUE; 483 | nextEntry.values = Arrays.asList(readStringEncoded()); 484 | } 485 | 486 | private void readList() throws IOException { 487 | long len = readLength(); 488 | if (len > Integer.MAX_VALUE) { 489 | throw new IllegalArgumentException("Lists with more than " + Integer.MAX_VALUE 490 | + " elements are not supported."); 491 | } 492 | int size = (int)len; 493 | List list = new ArrayList(size); 494 | for (int i = 0; i < size; ++i) { 495 | list.add(readStringEncoded()); 496 | } 497 | nextEntry.valueType = ValueType.LIST; 498 | nextEntry.values = list; 499 | } 500 | 501 | private void readSet() throws IOException { 502 | long len = readLength(); 503 | if (len > Integer.MAX_VALUE) { 504 | throw new IllegalArgumentException("Sets with more than " + Integer.MAX_VALUE 505 | + " elements are not supported."); 506 | } 507 | int size = (int)len; 508 | List set = new ArrayList(size); 509 | for (int i = 0; i < size; ++i) { 510 | set.add(readStringEncoded()); 511 | } 512 | nextEntry.valueType = ValueType.SET; 513 | nextEntry.values = set; 514 | } 515 | 516 | private void readSortedSet() throws IOException { 517 | long len = readLength(); 518 | if (len > (Integer.MAX_VALUE / 2)) { 519 | throw new IllegalArgumentException("SortedSets with more than " + (Integer.MAX_VALUE / 2) 520 | + " elements are not supported."); 521 | } 522 | int size = (int)len; 523 | List valueScoresPairs = new ArrayList(2 * size); 524 | for (int i = 0; i < size; ++i) { 525 | valueScoresPairs.add(readStringEncoded()); 526 | valueScoresPairs.add(readDoubleString()); 527 | } 528 | nextEntry.valueType = ValueType.SORTED_SET; 529 | nextEntry.values = valueScoresPairs; 530 | } 531 | 532 | private void readSortedSet2() throws IOException { 533 | long len = readLength(); 534 | if (len > (Integer.MAX_VALUE / 2)) { 535 | throw new IllegalArgumentException("SortedSets with more than " + (Integer.MAX_VALUE / 2) 536 | + " elements are not supported."); 537 | } 538 | int size = (int)len; 539 | List valueScoresPairs = new ArrayList(2 * size); 540 | for (int i = 0; i < size; ++i) { 541 | valueScoresPairs.add(readStringEncoded()); 542 | valueScoresPairs.add(readBytes(8)); 543 | } 544 | nextEntry.valueType = ValueType.SORTED_SET2; 545 | nextEntry.values = valueScoresPairs; 546 | } 547 | 548 | private void readHash() throws IOException { 549 | long len = readLength(); 550 | if (len > (Integer.MAX_VALUE / 2)) { 551 | throw new IllegalArgumentException("Hashes with more than " + (Integer.MAX_VALUE / 2) 552 | + " elements are not supported."); 553 | } 554 | int size = (int)len; 555 | List kvPairs = new ArrayList(2 * size); 556 | for (int i = 0; i < size; ++i) { 557 | kvPairs.add(readStringEncoded()); 558 | kvPairs.add(readStringEncoded()); 559 | } 560 | nextEntry.valueType = ValueType.HASH; 561 | nextEntry.values = kvPairs; 562 | } 563 | 564 | private void readZipMap() throws IOException { 565 | nextEntry.valueType = ValueType.ZIPMAP; 566 | nextEntry.values = new ZipMap(readStringEncoded()); 567 | } 568 | 569 | private void readZipList() throws IOException { 570 | nextEntry.valueType = ValueType.ZIPLIST; 571 | nextEntry.values = new ZipList(readStringEncoded()); 572 | } 573 | 574 | private void readIntSet() throws IOException { 575 | nextEntry.valueType = ValueType.INTSET; 576 | nextEntry.values = new IntSet(readStringEncoded()); 577 | } 578 | 579 | private void readSortedSetAsZipList() throws IOException { 580 | nextEntry.valueType = ValueType.SORTED_SET_AS_ZIPLIST; 581 | nextEntry.values = new SortedSetAsZipList(readStringEncoded()); 582 | } 583 | 584 | private void readHashmapAsZipList() throws IOException { 585 | nextEntry.valueType = ValueType.HASHMAP_AS_ZIPLIST; 586 | nextEntry.values = new ZipList(readStringEncoded()); 587 | } 588 | 589 | private void readHashListPack() throws IOException { 590 | nextEntry.valueType = ValueType.HASHMAP_AS_LISTPACK; 591 | nextEntry.values = new ListpackList(readStringEncoded()); 592 | } 593 | 594 | private void readQuickList2() throws IOException { 595 | int size = (int)readLength(); 596 | List listpacks = new ArrayList(size); 597 | for (int i = 0; i < size; ++i) { 598 | // Throw away container format 599 | readLength(); 600 | listpacks.add(readStringEncoded()); 601 | } 602 | nextEntry.valueType = ValueType.QUICKLIST2; 603 | nextEntry.values = new QuickList2(listpacks); 604 | } 605 | 606 | private void readQuickList() throws IOException { 607 | long len = readLength(); 608 | if (len > Integer.MAX_VALUE) { 609 | throw new IllegalArgumentException("Quicklists with more than " + Integer.MAX_VALUE 610 | + " nested Ziplists are not supported."); 611 | } 612 | int size = (int)len; 613 | List ziplists = new ArrayList(size); 614 | for (int i = 0; i < size; ++i) { 615 | ziplists.add(readStringEncoded()); 616 | } 617 | nextEntry.valueType = ValueType.QUICKLIST; 618 | nextEntry.values = new QuickList(ziplists); 619 | } 620 | 621 | private void readHashMetadata(boolean gaType) throws IOException { 622 | if (gaType) { 623 | nextEntry.valueType = ValueType.HASHMAP_WITH_METADATA; 624 | nextEntry.minHashExpireTime = readExpirationMillis(); 625 | } else { 626 | nextEntry.valueType = ValueType.HASHMAP_WITH_METADATA_PRE_GA; 627 | } 628 | long len = readLength(); 629 | if (len > (Integer.MAX_VALUE / 3)) { 630 | throw new IllegalArgumentException("Hashes with metadata more than " 631 | + (Integer.MAX_VALUE / 3) 632 | + " elements are not supported."); 633 | } 634 | int size = (int)len; 635 | List kvxTuples = new ArrayList(3 * size); 636 | for (int i = 0; i < size; ++i) { 637 | long hashExpiry = readLength(); 638 | if (hashExpiry > 0) { 639 | hashExpiry += nextEntry.getMinHashExpireTime() - 1; 640 | } 641 | kvxTuples.add(readStringEncoded()); 642 | kvxTuples.add(readStringEncoded()); 643 | kvxTuples.add(String.valueOf(hashExpiry).getBytes(ASCII)); 644 | } 645 | 646 | nextEntry.values = kvxTuples; 647 | } 648 | 649 | private void readHashListPackEx(boolean gaType) throws IOException { 650 | if (gaType) { 651 | nextEntry.valueType = ValueType.HASHMAP_AS_LISTPACK_EX; 652 | nextEntry.minHashExpireTime = readExpirationMillis(); 653 | } else { 654 | nextEntry.valueType = ValueType.HASHMAP_AS_LISTPACK_EX_PRE_GA; 655 | } 656 | nextEntry.values = new ListpackList(readStringEncoded()); 657 | } 658 | 659 | /** 660 | * Closes the underlying file or stream. 661 | * 662 | * @throws IOException from closing the underlying channel. 663 | */ 664 | @Override 665 | public void close() throws IOException { 666 | ch.close(); 667 | } 668 | 669 | /** 670 | * Parses the raw score of an element in a {@link ValueType#SORTED_SET} or 671 | * {@link ValueType#SORTED_SET_AS_ZIPLIST}. 672 | */ 673 | public static double parseSortedSetScore(byte[] bytes) { 674 | return Double.parseDouble(new String(bytes, ASCII)); 675 | } 676 | 677 | /** 678 | * Parses the raw score of an element in a {@link ValueType#SORTED_SET2}. 679 | */ 680 | public static double parseSortedSet2Score(byte[] bytes) { 681 | return ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getDouble(); 682 | } 683 | } 684 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/ResizeDb.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | /** 16 | *

Resize DB entries contain information to speed up RDB loading by avoiding additional resizes 17 | * and rehashing. 18 | * 19 | *

Specifically, it contains the following: 20 | *

24 | * 25 | *

Introduced in RDB version 7. 26 | * 27 | * @author John Whitbeck 28 | */ 29 | public final class ResizeDb implements Entry { 30 | 31 | private final long dbHashTableSize; 32 | private final long expireTimeHashTableSize; 33 | 34 | ResizeDb(long dbHashTableSize, long expireTimeHashTableSize) { 35 | this.dbHashTableSize = dbHashTableSize; 36 | this.expireTimeHashTableSize = expireTimeHashTableSize; 37 | } 38 | 39 | @Override 40 | public EntryType getType() { 41 | return EntryType.RESIZE_DB; 42 | } 43 | 44 | /** 45 | * Returns the size of the DB hash table. 46 | * 47 | * @return size of the DB hash table. 48 | */ 49 | public long getDbHashTableSize() { 50 | return dbHashTableSize; 51 | } 52 | 53 | /** 54 | * Returns the size of the expire time hash table. 55 | * 56 | * @return size of the expire time hash table. 57 | */ 58 | public long getExpireTimeHashTableSize() { 59 | return expireTimeHashTableSize; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return String.format("%s (db hash table size: %d, expire time hash table size: %d)", 65 | EntryType.RESIZE_DB, dbHashTableSize, expireTimeHashTableSize); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/SelectDb.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | /** 16 | *

DB selection entries mark the beginning of a new database in the RDB dump file. All subsequent 17 | * {@link KeyValuePair}s until the next {@link SelectDb} or {@link Eof} entry belong to this 18 | * database. 19 | * 20 | * @author John Whitbeck 21 | */ 22 | public final class SelectDb implements Entry { 23 | 24 | private final long id; 25 | 26 | SelectDb(long id) { 27 | this.id = id; 28 | } 29 | 30 | @Override 31 | public EntryType getType() { 32 | return EntryType.SELECT_DB; 33 | } 34 | 35 | /** 36 | * Returns the identifier of this database. 37 | * 38 | * @return the database identifier 39 | */ 40 | public long getId() { 41 | return id; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return EntryType.SELECT_DB + " (" + id + ")"; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/SortedSetAsListpack.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.nio.charset.Charset; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.ListIterator; 19 | 20 | final class SortedSetAsListpack extends LazyList { 21 | 22 | private static final Charset ASCII = Charset.forName("ASCII"); 23 | 24 | private static final byte[] POS_INF_BYTES = "inf".getBytes(ASCII); 25 | private static final byte[] NEG_INF_BYTES = "-inf".getBytes(ASCII); 26 | private static final byte[] NAN_BYTES = "nan".getBytes(ASCII); 27 | 28 | private final byte[] envelope; 29 | 30 | SortedSetAsListpack(byte[] envelope) { 31 | this.envelope = envelope; 32 | } 33 | 34 | @Override 35 | protected List realize() { 36 | List values = new ListpackList(envelope).realize(); 37 | // fix the "+inf", "-inf", and "nan" values 38 | for (ListIterator i = values.listIterator(); i.hasNext(); ) { 39 | byte[] val = i.next(); 40 | if (Arrays.equals(val, POS_INF_BYTES)) { 41 | i.set(DoubleBytes.POSITIVE_INFINITY); 42 | } else if (Arrays.equals(val, NEG_INF_BYTES)) { 43 | i.set( DoubleBytes.NEGATIVE_INFINITY); 44 | } else if (Arrays.equals(val, NAN_BYTES)) { 45 | i.set(DoubleBytes.NaN); 46 | } 47 | } 48 | return values; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/SortedSetAsZipList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.nio.charset.Charset; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.ListIterator; 19 | 20 | final class SortedSetAsZipList extends LazyList { 21 | 22 | private static final Charset ASCII = Charset.forName("ASCII"); 23 | 24 | private static final byte[] POS_INF_BYTES = "inf".getBytes(ASCII); 25 | private static final byte[] NEG_INF_BYTES = "-inf".getBytes(ASCII); 26 | private static final byte[] NAN_BYTES = "nan".getBytes(ASCII); 27 | 28 | private final byte[] envelope; 29 | 30 | SortedSetAsZipList(byte[] envelope) { 31 | this.envelope = envelope; 32 | } 33 | 34 | @Override 35 | protected List realize() { 36 | List values = new ZipList(envelope).realize(); 37 | // fix the "+inf", "-inf", and "nan" values 38 | for (ListIterator i = values.listIterator(); i.hasNext(); ) { 39 | byte[] val = i.next(); 40 | if (Arrays.equals(val, POS_INF_BYTES)) { 41 | i.set(DoubleBytes.POSITIVE_INFINITY); 42 | } else if (Arrays.equals(val, NEG_INF_BYTES)) { 43 | i.set( DoubleBytes.NEGATIVE_INFINITY); 44 | } else if (Arrays.equals(val, NAN_BYTES)) { 45 | i.set(DoubleBytes.NaN); 46 | } 47 | } 48 | return values; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/StringUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | final class StringUtils { 16 | 17 | static String getPrintableString(byte[] bytes) { 18 | StringBuilder sb = new StringBuilder(); 19 | for (byte b : bytes) { 20 | if (b > 31 && b < 127) { // printable ascii characters 21 | sb.append((char)b); 22 | } else { 23 | sb.append(String.format("\\x%02x", (int)b & 0xff)); 24 | } 25 | } 26 | return sb.toString(); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/ValueType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | 16 | /** 17 | * This enum holds the different value serialization type encountered in an RDB file. 18 | * 19 | * @author John Whitbeck 20 | */ 21 | public enum ValueType { 22 | 23 | /** 24 | * A simple redis key/value pair as created by set foo bar. 25 | */ 26 | VALUE, 27 | 28 | /** 29 | * A redis list as created by lpush foo bar. 30 | */ 31 | LIST, 32 | 33 | /** 34 | *A redis set as created by sadd foo bar. 35 | */ 36 | SET, 37 | 38 | /** 39 | * A redis sorted set as created by zadd foo 1.2 bar. 40 | */ 41 | SORTED_SET, 42 | 43 | /** 44 | * A redis hash as created by hset foo bar baz. 45 | */ 46 | HASH, 47 | 48 | /** 49 | * A compact encoding for small hashes. Deprecated as of redis 2.6. 50 | */ 51 | ZIPMAP, 52 | 53 | /** 54 | * A compact encoding for small lists. 55 | */ 56 | ZIPLIST, 57 | 58 | /** 59 | * A compact encoding for sets comprised entirely of integers. 60 | */ 61 | INTSET, 62 | 63 | /** 64 | * A compact encoding for small sorted sets in which value/score pairs are flattened and stored in 65 | * a ZipList. 66 | */ 67 | SORTED_SET_AS_ZIPLIST, 68 | 69 | /** 70 | * A compact encoding for small sorted sets in which value/score pairs are flattened and stored in 71 | * a ZipList. 72 | */ 73 | SORTED_SET_AS_LISTPACK, 74 | 75 | /** 76 | * A compact encoding for small hashes in which key/value pairs are flattened and stored in a 77 | * ZipList. 78 | */ 79 | HASHMAP_AS_ZIPLIST, 80 | 81 | /** 82 | * A linked list of ziplists to achieve good compression on lists of any length. 83 | */ 84 | QUICKLIST, 85 | 86 | /** 87 | * A linked list of listpacks to achieve good compression on lists of any length. 88 | */ 89 | QUICKLIST2, 90 | 91 | /** 92 | * Like SORTED_SET but encodes the scores as doubles using the IEEE 754 floating-point "double 93 | * format" bit layout on 8 bits instead of a string representation of the score. 94 | */ 95 | SORTED_SET2, 96 | 97 | /** 98 | * A compact encoding for small hashes. Replaces HASHMAP_AS_ZIPLIST as of RDB 10. 99 | */ 100 | HASHMAP_AS_LISTPACK, 101 | 102 | /** 103 | * A compact encoding of elements. Replaces ZIPLIST in RDB 10. 104 | */ 105 | LISTPACK, 106 | 107 | /** 108 | * A compact encoding for small sets. 109 | */ 110 | SET_AS_LISTPACK, 111 | 112 | /** 113 | * A redis hash with expiration time on each hash key. Hash keys are stored 114 | * as a series of key, value, ttl tuples. 115 | */ 116 | HASHMAP_WITH_METADATA, 117 | 118 | /** 119 | * As HASHMAP_WITH_METADATA, but the pre-GA encoding stored the hash 120 | * expiration as actual times while the GA version stores the hash expiration 121 | * as an offset from the next key expiration. 122 | */ 123 | HASHMAP_WITH_METADATA_PRE_GA, 124 | 125 | /** 126 | * A compact encoding for small hashes with expiration time on each hash key. 127 | * Values are stored as a listpack with key, value, ttl tuples. 128 | */ 129 | HASHMAP_AS_LISTPACK_EX, 130 | 131 | /** 132 | * As HASHMAP_AS_LISTPACK_EX, but the pre-GA encoding stored the hash 133 | * expiration as actual times while the GA version stores the hash expiration 134 | * as an offset from the key expiration. 135 | */ 136 | HASHMAP_AS_LISTPACK_EX_PRE_GA; 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/ZipList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.nio.charset.Charset; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | final class ZipList extends LazyList { 20 | 21 | private static final Charset ASCII = Charset.forName("ASCII"); 22 | private final byte[] envelope; 23 | 24 | ZipList(byte[] envelope) { 25 | this.envelope = envelope; 26 | } 27 | 28 | @Override 29 | protected List realize() { 30 | // skip the first 8 bytes representing the total size in bytes of the ziplist and the offset to 31 | // the last element. 32 | int pos = 8; 33 | // read number of elements as a 2 byte little-endian integer 34 | int num = ((int)envelope[pos++] & 0xff) << 0 35 | | ((int)envelope[pos++] & 0xff) << 8; 36 | List list = new ArrayList(num); 37 | int idx = 0; 38 | while (idx < num) { 39 | // skip length of previous entry. If len is <= 253 (0xfd), it represents the length of the 40 | // previous entry, otherwise, the next four bytes are used to store the length 41 | int prevLen = (int)envelope[pos++] & 0xff; 42 | if (prevLen > 0xfd) { 43 | pos += 4; 44 | } 45 | int special = (int)envelope[pos++] & 0xff; 46 | int top2bits = special >> 6; 47 | int len; 48 | byte[] buf; 49 | switch (top2bits) { 50 | case 0: // string value with length less than or equal to 63 bytes (6 bits) 51 | len = special & 0x3f; 52 | buf = new byte[len]; 53 | System.arraycopy(envelope, pos, buf, 0, len); 54 | pos += len; 55 | list.add(buf); 56 | break; 57 | case 1: // String value with length less than or equal to 16383 bytes (14 bits). 58 | len = ((special & 0x3f) << 8) | ((int)envelope[pos++] & 0xff); 59 | buf = new byte[len]; 60 | System.arraycopy(envelope, pos, buf, 0, len); 61 | pos += len; 62 | list.add(buf); 63 | break; 64 | case 2: /* String value with length greater than or equal to 16384 bytes. Length is read 65 | from 4 following bytes. */ 66 | len = ((int)envelope[pos++] & 0xff) << 24 67 | | ((int)envelope[pos++] & 0xff) << 16 68 | | ((int)envelope[pos++] & 0xff) << 8 69 | | ((int)envelope[pos++] & 0xff) << 0; 70 | buf = new byte[len]; 71 | System.arraycopy(envelope, pos, buf, 0, len); 72 | pos += len; 73 | list.add(buf); 74 | break; 75 | case 3: // integer encodings 76 | int flag = (special & 0x30) >> 4; 77 | long val; 78 | switch (flag) { 79 | case 0: // read next 2 bytes as a 16 bit signed integer 80 | val = (long)envelope[pos++] & 0xff 81 | | (long)envelope[pos++] << 8; 82 | list.add(String.valueOf(val).getBytes(ASCII)); 83 | break; 84 | case 1: // read next 4 bytes as a 32 bit signed integer 85 | val = ((long)envelope[pos++] & 0xff) << 0 86 | | ((long)envelope[pos++] & 0xff) << 8 87 | | ((long)envelope[pos++] & 0xff) << 16 88 | | (long)envelope[pos++] << 24; 89 | list.add(String.valueOf(val).getBytes(ASCII)); 90 | break; 91 | case 2: // read next 8 as a 64 bit signed integer 92 | val = ((long)envelope[pos++] & 0xff) << 0 93 | | ((long)envelope[pos++] & 0xff) << 8 94 | | ((long)envelope[pos++] & 0xff) << 16 95 | | ((long)envelope[pos++] & 0xff) << 24 96 | | ((long)envelope[pos++] & 0xff) << 32 97 | | ((long)envelope[pos++] & 0xff) << 40 98 | | ((long)envelope[pos++] & 0xff) << 48 99 | | (long)envelope[pos++] << 56; 100 | list.add(String.valueOf(val).getBytes(ASCII)); 101 | break; 102 | case 3: 103 | int loBits = special & 0x0f; 104 | switch (loBits) { 105 | case 0: // read next 3 bytes as a 24 bit signed integer 106 | val = ((long)envelope[pos++] & 0xff) << 0 107 | | ((long)envelope[pos++] & 0xff) << 8 108 | | (long)envelope[pos++] << 16; 109 | list.add(String.valueOf(val).getBytes(ASCII)); 110 | break; 111 | case 0x0e: // read next byte as an 8 bit signed integer 112 | val = (long)envelope[pos++]; 113 | list.add(String.valueOf(val).getBytes(ASCII)); 114 | break; 115 | default: /* an immediate 4 bit unsigned integer between 0 and 12. Substract 1 as the 116 | range is actually between 1 and 13. */ 117 | list.add(String.valueOf(loBits - 1).getBytes(ASCII)); 118 | break; 119 | } 120 | break; 121 | default: // never reached 122 | } 123 | break; 124 | default: // never reached 125 | } 126 | idx += 1; 127 | } 128 | return list; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/ZipMap.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | package net.whitbeck.rdbparser; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | final class ZipMap extends LazyList { 19 | 20 | private final byte[] envelope; 21 | 22 | ZipMap(byte[] envelope) { 23 | this.envelope = envelope; 24 | } 25 | 26 | @Override 27 | protected List realize() { 28 | // The structure of the zip map is: 29 | // "foo""bar""hello""world" 30 | 31 | int pos = 0; 32 | // The first byte holds the size of the zip map. If it is greater than or equal to 254, 33 | // value is not used and we will have to iterate the entire zip map to find the length. 34 | int zmlen = (int)envelope[pos++] & 0xff; 35 | List list = zmlen < 254 ? new ArrayList(2*zmlen) : new ArrayList(); 36 | while (true) { 37 | int b = (int)envelope[pos++] & 0xff; 38 | if (b == 255) { // reached end of zipmap 39 | break; 40 | } 41 | // Read a key/value pair. 42 | 43 | // Read the length of the following string, which can be either a key or a value. This length 44 | // is stored in either 1 byte or 5 bytes. If the first byte is between 0 and 252, that is the 45 | // length of the value. If the first byte is 253, then the next 4 bytes read as an unsigned 46 | // integer represent the length of the zipmap. 254 and 255 are invalid values for this field. 47 | int len = 0; 48 | if (b < 253) { 49 | len = b; 50 | } else { 51 | len = ((int)envelope[pos++] & 0xff) << 24 52 | | ((int)envelope[pos++] & 0xff) << 16 53 | | ((int)envelope[pos++] & 0xff) << 8 54 | | ((int)envelope[pos++] & 0xff) << 0; 55 | } 56 | // Read the key. 57 | byte[] buf = new byte[len]; 58 | System.arraycopy(envelope, pos, buf, 0, len); 59 | pos += len; 60 | list.add(buf); 61 | 62 | // Read the length of the value (similar to the length of the key). 63 | b = (int)envelope[pos++] & 0xff; 64 | len = 0; 65 | if (b < 253) { 66 | len = b; 67 | } else { 68 | len = ((int)envelope[pos++] & 0xff) << 24 69 | | ((int)envelope[pos++] & 0xff) << 16 70 | | ((int)envelope[pos++] & 0xff) << 8 71 | | ((int)envelope[pos++] & 0xff) << 0; 72 | } 73 | // Read the number of free bytes after the value. This is always 1 byte.For example, if the 74 | // value of a key is “America” and its get updated to “USA”, 4 free bytes will be available. 75 | int free = (int)envelope[pos++] & 0xff; 76 | // Read the value. 77 | buf = new byte[len]; 78 | System.arraycopy(envelope, pos, buf, 0, len); 79 | pos += len + free; 80 | list.add(buf); 81 | } 82 | return list; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/net/whitbeck/rdbparser/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | *

The use and distribution terms for this software are covered by the 5 | * Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) 6 | * which can be found in the file al-v20.txt at the root of this distribution. 7 | * By using this software in any fashion, you are agreeing to be bound by 8 | * the terms of this license. 9 | * 10 | *

You must not remove this notice, or any other, from this software. 11 | */ 12 | 13 | /** 14 | * Provides a simple Redis RDB file parser for Java. 15 | * 16 | *

This library does the minimal amount of work to read entries (e.g. a new DB selector, or a 17 | * key/value pair with an expire time) from an RDB file, mostly limiting itself to returning byte 18 | * arrays or lists of byte arrays for keys and values. The caller is responsible for 19 | * application-level decisions like how to interpret the contents of the returned byte arrays or 20 | * what types of objects to instantiate from them. 21 | * 22 | *

For example, sorted sets and hashes are parsed as a flat list of value/score pairs and 23 | * key/value pairs, respectively. Simple Redis values are parsed as a singleton. As expected, Redis 24 | * lists and sets are parsed as lists of values. 25 | * 26 | *

Furthermore, this library performs lazy decoding of the packed encodings (ZipMap, ZipList, 27 | * Hashmap as ZipList, Sorted Set as ZipList, Intset, and Quicklist) such that those are only 28 | * decoded when needed. This allows the caller to efficiently skip over these entries or defer their 29 | * decoding to a worker thread. 30 | * 31 | *

RDB files created by all versions of Redis through 7.4.x are supported (i.e., RDB versions 1 32 | * through 12). Some features, however, are not supported: 33 | * 34 | *

38 | * 39 | *

If you need them, please open an issue or a pull request. 40 | * 41 | *

Valkey uses the same RDB format as of 8.0.x, and this library can read those as well. 42 | * 43 | *

Implementation is not thread safe. 44 | * 45 | *

As of November 2024, the most recent RDB format version is 12. The source of truth is the rdb.h file in the Redis repo. The following resources provide a good 48 | * overview of the RDB format. 49 | * 50 | *

67 | */ 68 | package net.whitbeck.rdbparser; 69 | -------------------------------------------------------------------------------- /src/test/java/net/whitbeck/rdbparser/RdbParserTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-2021 John Whitbeck. All rights reserved. 3 | * 4 | * The use and distribution terms for this software are covered by the Apache License 2.0 5 | * (https://www.apache.org/licenses/LICENSE-2.0.txt) which can be found in the file al-v20.txt at 6 | * the root of this distribution. By using this software in any fashion, you are agreeing to be 7 | * bound by the terms of this license. 8 | * 9 | * You must not remove this notice, or any other, from this software. 10 | */ 11 | 12 | package net.whitbeck.rdbparser; 13 | 14 | import java.io.File; 15 | import java.io.IOException; 16 | import java.nio.ByteBuffer; 17 | import java.nio.channels.FileChannel; 18 | import java.nio.file.Files; 19 | import java.nio.file.StandardOpenOption; 20 | import java.util.ArrayList; 21 | import java.util.Arrays; 22 | import java.util.Collection; 23 | import java.util.Collections; 24 | import java.util.HashMap; 25 | import java.util.HashSet; 26 | import java.util.Iterator; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.Set; 30 | import java.util.regex.Pattern; 31 | 32 | import org.junit.AfterClass; 33 | import org.junit.Assert; 34 | import org.junit.BeforeClass; 35 | import org.junit.Rule; 36 | import org.junit.Test; 37 | import org.junit.runner.RunWith; 38 | import org.junit.runners.Parameterized; 39 | import org.junit.runners.Parameterized.Parameters; 40 | import org.junit.rules.ExpectedException; 41 | 42 | import redis.clients.jedis.Jedis; 43 | 44 | @RunWith(Parameterized.class) 45 | public class RdbParserTest { 46 | 47 | static final class RedisServerInstance { 48 | static int nextPort = 4000; 49 | 50 | final int rdbVersion; 51 | final String redisVersion; 52 | final int port; 53 | final File workDir; 54 | final File dumpFile; 55 | Process proc; 56 | Jedis jedis; 57 | 58 | RedisServerInstance(String redisVersion, int rdbVersion) { 59 | this.redisVersion = redisVersion; 60 | this.rdbVersion = rdbVersion; 61 | port = ++nextPort; 62 | workDir = new File("redis", "" + redisVersion); 63 | dumpFile = new File(workDir, "dump.rdb"); 64 | } 65 | 66 | void startRedisServer() throws Exception { 67 | dumpFile.delete(); // start from an empty dump 68 | File configFile = new File(workDir, "src/configs/redis.conf"); 69 | configFile.getParentFile().mkdirs(); 70 | Files.write(configFile.toPath(), bytes("port " + port + "\n")); 71 | proc = new ProcessBuilder().directory(workDir) 72 | .command("src/redis-server", configFile.getCanonicalPath()) 73 | .start(); 74 | } 75 | 76 | void startJedis() throws Exception { 77 | jedis = new Jedis("localhost", port); 78 | } 79 | 80 | void stop() throws Exception { 81 | jedis.close(); 82 | proc.destroy(); 83 | dumpFile.delete(); 84 | } 85 | } 86 | 87 | static final RedisServerInstance[] instances = new RedisServerInstance[] { 88 | new RedisServerInstance("2.8.24", 6), 89 | new RedisServerInstance("3.2.11", 7), 90 | new RedisServerInstance("4.0.6", 8), 91 | new RedisServerInstance("5.0.14", 9), 92 | new RedisServerInstance("6.2.1", 9), 93 | new RedisServerInstance("7.0.11", 10), 94 | new RedisServerInstance("7.2.4", 11), 95 | new RedisServerInstance("7.4.1", 12), 96 | }; 97 | 98 | @BeforeClass 99 | public static void startClients() throws Exception { 100 | for (RedisServerInstance inst : instances) { 101 | inst.startRedisServer(); 102 | } 103 | Thread.sleep(2000); // wait for the redis servers to start 104 | for (RedisServerInstance inst : instances) { 105 | inst.startJedis(); 106 | } 107 | } 108 | 109 | @AfterClass 110 | public static void stopClients() throws Exception { 111 | for (RedisServerInstance inst : instances) { 112 | inst.stop(); 113 | } 114 | } 115 | 116 | int rdbVersion; 117 | String redisVersion; 118 | File dumpFile; 119 | Jedis jedis; 120 | 121 | // accept 5s difference for absolute timestamp comparisons 122 | private static final int EXPIRATION_TOLERANCE_MS = 5000; 123 | Map originalConfig; 124 | 125 | @Parameters 126 | public static Collection params() { 127 | List params = new ArrayList(instances.length); 128 | for (RedisServerInstance inst : instances) { 129 | params.add(new Object[] {inst}); 130 | } 131 | return params; 132 | } 133 | 134 | public RdbParserTest(RedisServerInstance inst) { 135 | this.rdbVersion = inst.rdbVersion; 136 | this.redisVersion = inst.redisVersion; 137 | this.dumpFile = inst.dumpFile; 138 | this.jedis = inst.jedis; 139 | } 140 | 141 | 142 | @Rule 143 | public ExpectedException thrown = ExpectedException.none(); 144 | 145 | RdbParser openTestParser() throws IOException { 146 | return new RdbParser(dumpFile); 147 | } 148 | 149 | static String str(byte[] bs) throws Exception { 150 | return new String(bs, "ASCII"); 151 | } 152 | 153 | static byte[] bytes(String s) throws Exception { 154 | return s.getBytes("ASCII"); 155 | } 156 | 157 | static int compareVersions(String v1, String v2) { 158 | String[] elems1 = v1.split("\\."); 159 | String[] elems2 = v2.split("\\."); 160 | int major1 = Integer.parseInt(elems1[0]); 161 | int major2 = Integer.parseInt(elems2[0]); 162 | if (major1 > major2) { 163 | return 1; 164 | } else if (major1 < major2) { 165 | return -1; 166 | } else { 167 | int minor1 = Integer.parseInt(elems1[1]); 168 | int minor2 = Integer.parseInt(elems2[1]); 169 | if (minor1 > minor2) { 170 | return 1; 171 | } else if (minor1 < minor2) { 172 | return -1; 173 | } else { 174 | int bugfix1 = Integer.parseInt(elems1[2]); 175 | int bugfix2 = Integer.parseInt(elems2[2]); 176 | if (bugfix1 > bugfix2) { 177 | return 1; 178 | } else if (bugfix1 < bugfix2) { 179 | return -1; 180 | } else { 181 | return 0; 182 | } 183 | } 184 | } 185 | } 186 | 187 | static boolean isEarlierThan(String version, String cmp) { 188 | return compareVersions(version, cmp) < 0; 189 | } 190 | 191 | static boolean isLaterThan(String version, String cmp) { 192 | return compareVersions(version, cmp) > 0; 193 | } 194 | 195 | private void setTestConfig(String key, String value) { 196 | if (originalConfig == null) { 197 | originalConfig = jedis.configGet("*"); 198 | } 199 | jedis.configSet(key, value); 200 | } 201 | 202 | private void restoreConfig(String key) { 203 | if (originalConfig == null) { 204 | return; 205 | } 206 | jedis.configSet(key, originalConfig.get(key)); 207 | } 208 | 209 | void setTestFile(ByteBuffer buf) throws IOException { 210 | try (FileChannel ch = FileChannel.open(dumpFile.toPath(), StandardOpenOption.WRITE, 211 | StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { 212 | ch.write(buf); 213 | } 214 | } 215 | 216 | @Test 217 | public void magicNumber() throws Exception { 218 | setTestFile(ByteBuffer.wrap(bytes("not a valid RDB file"))); 219 | try (RdbParser p = openTestParser()) { 220 | thrown.expect(IllegalStateException.class); 221 | thrown.expectMessage("Not a valid redis RDB file"); 222 | p.readNext(); 223 | } 224 | } 225 | 226 | @Test 227 | public void versionCheck() throws Exception { 228 | setTestFile(ByteBuffer.wrap(bytes("REDIS0042"))); 229 | try (RdbParser p = openTestParser()) { 230 | thrown.expect(IllegalStateException.class); 231 | thrown.expectMessage("Unknown version"); 232 | p.readNext(); 233 | } 234 | } 235 | 236 | @Test 237 | public void emptyFile() throws Exception { 238 | jedis.flushAll(); 239 | jedis.save(); 240 | try (RdbParser p = openTestParser()) { 241 | Entry t; 242 | AuxField aux; 243 | if (rdbVersion >= 7) { 244 | // AUX redis-ver = 3.2.0 245 | t = p.readNext(); 246 | Assert.assertEquals(EntryType.AUX_FIELD, t.getType()); 247 | aux = (AuxField) t; 248 | Assert.assertArrayEquals(bytes("redis-ver"), aux.getKey()); 249 | Assert.assertArrayEquals(bytes(redisVersion), aux.getValue()); 250 | // AUX redis-bits = 64 251 | t = p.readNext(); 252 | Assert.assertEquals(EntryType.AUX_FIELD, t.getType()); 253 | aux = (AuxField) t; 254 | Assert.assertArrayEquals(bytes("redis-bits"), aux.getKey()); 255 | Assert.assertArrayEquals(bytes("64"), aux.getValue()); 256 | // AUX ctime 257 | t = p.readNext(); 258 | Assert.assertEquals(EntryType.AUX_FIELD, t.getType()); 259 | aux = (AuxField) t; 260 | Assert.assertArrayEquals(bytes("ctime"), aux.getKey()); 261 | // AUX used-mem = 821176 262 | t = p.readNext(); 263 | Assert.assertEquals(EntryType.AUX_FIELD, t.getType()); 264 | aux = (AuxField) t; 265 | Assert.assertArrayEquals(bytes("used-mem"), aux.getKey()); 266 | } 267 | if (rdbVersion >= 8) { 268 | String aofField = "aof-preamble"; 269 | if (rdbVersion >= 10) { 270 | aofField = "aof-base"; 271 | } 272 | t = p.readNext(); 273 | Assert.assertEquals(EntryType.AUX_FIELD, t.getType()); 274 | aux = (AuxField) t; 275 | Assert.assertArrayEquals(bytes(aofField), aux.getKey()); 276 | Assert.assertArrayEquals(bytes("0"), aux.getValue()); 277 | } 278 | // EOF 279 | t = p.readNext(); 280 | Assert.assertEquals(EntryType.EOF, t.getType()); 281 | // Nothing after EOF 282 | Assert.assertNull(p.readNext()); 283 | } 284 | } 285 | 286 | void skipAuxFields(RdbParser p) throws Exception { 287 | // Skip the AUX entries at the beginning of the file. 288 | if (rdbVersion < 7) { 289 | return; 290 | } 291 | int numEntries = 4; 292 | if (rdbVersion >= 8 && rdbVersion < 10) { 293 | numEntries++; 294 | } 295 | for (int i = 0; i < numEntries; i++) { 296 | p.readNext(); 297 | } 298 | if (rdbVersion >= 10) { 299 | // Version 12 can have various additional AUX fields depending on the 300 | // server configuration, but aof-base is always the last one. 301 | while (true) { 302 | AuxField aux = (AuxField) p.readNext(); 303 | String key = str(aux.getKey()); 304 | if (key.equals("aof-base")) { 305 | break; 306 | } 307 | } 308 | } 309 | } 310 | 311 | @Test 312 | public void SelectDb() throws Exception { 313 | jedis.flushAll(); 314 | jedis.select(1); 315 | jedis.set("foo", "bar"); 316 | jedis.select(0); 317 | jedis.set("foo", "baz"); 318 | jedis.save(); 319 | try (RdbParser p = openTestParser()) { 320 | Entry t; 321 | ResizeDb resizeDb; 322 | SelectDb selectDb; 323 | KeyValuePair kvp; 324 | skipAuxFields(p); 325 | // DB SELECTOR 0 326 | t = p.readNext(); 327 | Assert.assertEquals(EntryType.SELECT_DB, t.getType()); 328 | selectDb = (SelectDb) t; 329 | Assert.assertEquals(0, selectDb.getId()); 330 | if (rdbVersion >= 7) { 331 | // Resize DB 332 | t = p.readNext(); 333 | Assert.assertEquals(EntryType.RESIZE_DB, t.getType()); 334 | resizeDb = (ResizeDb) t; 335 | Assert.assertEquals(1, resizeDb.getDbHashTableSize()); 336 | Assert.assertEquals(0, resizeDb.getExpireTimeHashTableSize()); 337 | } 338 | // foo:bar 339 | t = p.readNext(); 340 | Assert.assertEquals(EntryType.KEY_VALUE_PAIR, t.getType()); 341 | kvp = (KeyValuePair) t; 342 | Assert.assertEquals(ValueType.VALUE, kvp.getValueType()); 343 | Assert.assertEquals("foo", str(kvp.getKey())); 344 | Assert.assertEquals("baz", str(kvp.getValues().get(0))); 345 | // DB SELECTOR 1 346 | t = p.readNext(); 347 | Assert.assertEquals(EntryType.SELECT_DB, t.getType()); 348 | selectDb = (SelectDb) t; 349 | Assert.assertEquals(1, selectDb.getId()); 350 | if (rdbVersion >= 7) { 351 | // Resize DB 352 | t = p.readNext(); 353 | Assert.assertEquals(EntryType.RESIZE_DB, t.getType()); 354 | resizeDb = (ResizeDb) t; 355 | Assert.assertEquals(1, resizeDb.getDbHashTableSize()); 356 | Assert.assertEquals(0, resizeDb.getExpireTimeHashTableSize()); 357 | } 358 | // foo:baz 359 | t = p.readNext(); 360 | Assert.assertEquals(EntryType.KEY_VALUE_PAIR, t.getType()); 361 | kvp = (KeyValuePair) t; 362 | Assert.assertEquals(ValueType.VALUE, kvp.getValueType()); 363 | Assert.assertEquals("foo", str(kvp.getKey())); 364 | Assert.assertEquals("bar", str(kvp.getValues().get(0))); 365 | // EOF 366 | t = p.readNext(); 367 | Assert.assertEquals(EntryType.EOF, t.getType()); 368 | } 369 | } 370 | 371 | void skipToFirstKeyValuePair(RdbParser p) throws Exception { 372 | skipAuxFields(p); // Skip the AUX fields at top of file 373 | p.readNext(); // Skip the DB SELECTOR entry 374 | if (rdbVersion >= 7) { 375 | p.readNext(); // Skip the RESIZE_DB entry 376 | } 377 | } 378 | 379 | @Test 380 | public void expire() throws Exception { 381 | long expireTimeSecs = 3000000000L; 382 | long expireTimeMillis = 2000000000000L; 383 | int n = 0; 384 | jedis.flushAll(); 385 | jedis.set("noexpiretime", "val"); 386 | n++; 387 | jedis.set("seconds", "val"); 388 | jedis.expireAt("seconds", expireTimeSecs); 389 | n++; 390 | if (!isEarlierThan(redisVersion, "2.6.0")) { 391 | jedis.set("millis", "val"); 392 | jedis.pexpireAt("millis", expireTimeMillis); 393 | n++; 394 | } 395 | jedis.save(); 396 | try (RdbParser p = openTestParser()) { 397 | skipToFirstKeyValuePair(p); 398 | for (int i = 0; i < n; ++i) { 399 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 400 | String k = str(kvp.getKey()); 401 | if (k.equals("noexpiretime")) { 402 | Assert.assertNull(kvp.getExpireTime()); 403 | } else if (k.equals("seconds")) { 404 | Assert.assertEquals(expireTimeSecs * 1000L, kvp.getExpireTime().longValue()); 405 | } else if (k.equals("millis")) { 406 | Assert.assertEquals(expireTimeMillis, kvp.getExpireTime().longValue()); 407 | } 408 | } 409 | } 410 | } 411 | 412 | @Test 413 | public void stringRepresentations() throws Exception { 414 | jedis.flushAll(); 415 | jedis.set("simple-key", "val"); 416 | jedis.set("key-with-expire-time", "val"); 417 | long expireTime = 2000000000L; 418 | jedis.expireAt("key-with-expire-time", expireTime); 419 | jedis.lpush("list-key", "one", "two", "three"); 420 | jedis.set(new byte[] {0, 1, 2, 3}, bytes("val")); 421 | jedis.save(); 422 | try (RdbParser p = openTestParser()) { 423 | if (rdbVersion >= 7) { 424 | // AUX entries 425 | Assert.assertEquals("AUX_FIELD (k: redis-ver, v: " + redisVersion + ")", 426 | p.readNext().toString()); 427 | Assert.assertEquals("AUX_FIELD (k: redis-bits, v: 64)", p.readNext().toString()); 428 | Assert.assertTrue( 429 | Pattern.matches("AUX_FIELD \\(k: ctime, v: \\d{10}\\)", p.readNext().toString())); 430 | Assert.assertTrue( 431 | Pattern.matches("AUX_FIELD \\(k: used-mem, v: \\d+\\)", p.readNext().toString())); 432 | } 433 | if (rdbVersion >= 8) { 434 | // more AUX fields 435 | String aofField = "aof-preamble"; 436 | if (rdbVersion >= 10) { 437 | aofField = "aof-base"; 438 | } 439 | Assert.assertEquals("AUX_FIELD (k: " + aofField + ", v: 0)", p.readNext().toString()); 440 | } 441 | // DB 0 442 | Assert.assertEquals("SELECT_DB (0)", p.readNext().toString()); 443 | if (rdbVersion >= 7) { 444 | Assert.assertEquals("RESIZE_DB (db hash table size: 4, expire time hash table size: 1)", 445 | p.readNext().toString()); 446 | } 447 | for (int i = 0; i < 4; ++i) { 448 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 449 | byte[] k = kvp.getKey(); 450 | if (Arrays.equals(k, bytes("simple-key"))) { 451 | Assert.assertEquals("KEY_VALUE_PAIR (key: simple-key, 1 value)", kvp.toString()); 452 | } else if (Arrays.equals(k, bytes("key-with-expire-time"))) { 453 | Assert.assertEquals("KEY_VALUE_PAIR (key: key-with-expire-time, expire time: " 454 | + expireTime * 1000 + ", 1 value)", kvp.toString()); 455 | } else if (Arrays.equals(bytes("list-key"), k)) { 456 | Assert.assertEquals("KEY_VALUE_PAIR (key: list-key, 3 values)", kvp.toString()); 457 | } else if (Arrays.equals(k, new byte[] {0, 1, 2, 3})) { 458 | Assert.assertEquals("KEY_VALUE_PAIR (key: \\x00\\x01\\x02\\x03, 1 value)", 459 | kvp.toString()); 460 | } 461 | } 462 | Assert.assertTrue(Pattern.matches("EOF \\([\\da-f]{16}\\)", p.readNext().toString())); 463 | } 464 | } 465 | 466 | @Test 467 | public void binaryKeyAndValues() throws Exception { 468 | byte[] key = new byte[] {0, 1, 2, 3}; 469 | byte[] val = new byte[] {4, 5, 6}; 470 | jedis.flushAll(); 471 | jedis.set(key, val); 472 | jedis.save(); 473 | try (RdbParser p = openTestParser()) { 474 | skipToFirstKeyValuePair(p); 475 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 476 | Assert.assertArrayEquals(key, kvp.getKey()); 477 | Assert.assertArrayEquals(val, kvp.getValues().get(0)); 478 | } 479 | } 480 | 481 | @Test 482 | public void integerOneByteEncoding() throws Exception { 483 | jedis.flushAll(); 484 | jedis.set("foo", "12"); 485 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 486 | jedis.save(); 487 | try (RdbParser p = openTestParser()) { 488 | skipToFirstKeyValuePair(p); 489 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 490 | Assert.assertEquals("foo", str(kvp.getKey())); 491 | Assert.assertEquals("12", str(kvp.getValues().get(0))); 492 | } 493 | } 494 | 495 | @Test 496 | public void integerOneByteEncodingSigned() throws Exception { 497 | jedis.flushAll(); 498 | jedis.set("foo", "-12"); 499 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 500 | jedis.save(); 501 | try (RdbParser p = openTestParser()) { 502 | skipToFirstKeyValuePair(p); 503 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 504 | Assert.assertEquals("foo", str(kvp.getKey())); 505 | Assert.assertEquals("-12", str(kvp.getValues().get(0))); 506 | } 507 | } 508 | 509 | @Test 510 | public void integerTwoBytesEncoding() throws Exception { 511 | jedis.flushAll(); 512 | jedis.set("foo", "1234"); 513 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 514 | jedis.save(); 515 | try (RdbParser p = openTestParser()) { 516 | skipToFirstKeyValuePair(p); 517 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 518 | Assert.assertEquals("foo", str(kvp.getKey())); 519 | Assert.assertEquals("1234", str(kvp.getValues().get(0))); 520 | } 521 | } 522 | 523 | @Test 524 | public void integerTwoBytesEncodingSigned() throws Exception { 525 | jedis.flushAll(); 526 | jedis.set("foo", "-1234"); 527 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 528 | jedis.save(); 529 | try (RdbParser p = openTestParser()) { 530 | skipToFirstKeyValuePair(p); 531 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 532 | Assert.assertEquals("foo", str(kvp.getKey())); 533 | Assert.assertEquals("-1234", str(kvp.getValues().get(0))); 534 | } 535 | } 536 | 537 | @Test 538 | public void integerFourByteEncoding() throws Exception { 539 | jedis.flushAll(); 540 | jedis.set("foo", "123456789"); 541 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 542 | jedis.save(); 543 | try (RdbParser p = openTestParser()) { 544 | skipToFirstKeyValuePair(p); 545 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 546 | Assert.assertEquals("foo", str(kvp.getKey())); 547 | Assert.assertEquals("123456789", str(kvp.getValues().get(0))); 548 | } 549 | } 550 | 551 | @Test 552 | public void integerFourByteEncodingSigned() throws Exception { 553 | jedis.flushAll(); 554 | jedis.set("foo", "-123456789"); 555 | Assert.assertEquals("int", jedis.objectEncoding("foo")); 556 | jedis.save(); 557 | try (RdbParser p = openTestParser()) { 558 | skipToFirstKeyValuePair(p); 559 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 560 | Assert.assertEquals("foo", str(kvp.getKey())); 561 | Assert.assertEquals("-123456789", str(kvp.getValues().get(0))); 562 | } 563 | } 564 | 565 | @Test 566 | public void lzfEncoding() throws Exception { 567 | jedis.flushAll(); 568 | jedis.set("foo", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 569 | Assert.assertEquals("raw", jedis.objectEncoding("foo")); 570 | jedis.save(); 571 | try (RdbParser p = openTestParser()) { 572 | skipToFirstKeyValuePair(p); 573 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 574 | Assert.assertEquals("foo", str(kvp.getKey())); 575 | Assert.assertEquals( 576 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 577 | str(kvp.getValues().get(0))); 578 | } 579 | } 580 | 581 | @Test 582 | public void list() throws Exception { 583 | jedis.flushAll(); 584 | if (rdbVersion >= 10) { 585 | setTestConfig("list-max-listpack-size", "0"); 586 | } else if (rdbVersion >= 7) { 587 | setTestConfig("list-max-ziplist-size", "0"); 588 | } 589 | jedis.lpush("foo", "bar", "1234", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 590 | jedis.save(); 591 | if (rdbVersion >= 10) { 592 | restoreConfig("list-max-listpack-size"); 593 | } else if (rdbVersion >= 7) { 594 | restoreConfig("list-max-ziplist-size"); 595 | } 596 | try (RdbParser p = openTestParser()) { 597 | skipToFirstKeyValuePair(p); 598 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 599 | if (rdbVersion < 7) { 600 | Assert.assertEquals(ValueType.ZIPLIST, kvp.getValueType()); 601 | } else if (rdbVersion < 10) { 602 | Assert.assertEquals(ValueType.QUICKLIST, kvp.getValueType()); 603 | } else { 604 | Assert.assertEquals(ValueType.QUICKLIST2, kvp.getValueType()); 605 | } 606 | List list = kvp.getValues(); 607 | Assert.assertEquals("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", str(list.get(0))); 608 | Assert.assertEquals("1234", str(list.get(1))); 609 | Assert.assertEquals("bar", str(list.get(2))); 610 | } 611 | } 612 | 613 | @Test 614 | public void set() throws Exception { 615 | Set set = new HashSet(); 616 | Collections.addAll(set, "bar", "1234", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 617 | // Force set to use a non-listpack encoding. 618 | if (isLaterThan(redisVersion, "7.1.0")) { 619 | setTestConfig("set-max-listpack-value", "0"); 620 | } 621 | jedis.flushAll(); 622 | for (String elem : set) { 623 | jedis.sadd("foo", elem); 624 | } 625 | jedis.save(); 626 | if (isLaterThan(redisVersion, "7.1.0")) { 627 | restoreConfig("set-max-listpack-value"); 628 | } 629 | try (RdbParser p = openTestParser()) { 630 | skipToFirstKeyValuePair(p); 631 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 632 | Assert.assertEquals(ValueType.SET, kvp.getValueType()); 633 | Set parsedSet = new HashSet(); 634 | for (byte[] elem : kvp.getValues()) { 635 | parsedSet.add(str(elem)); 636 | } 637 | Assert.assertEquals(set, parsedSet); 638 | } 639 | } 640 | 641 | @Test 642 | public void sortedSet() throws Exception { 643 | if (rdbVersion < 8) { 644 | Map valueScoreMap = new HashMap(); 645 | valueScoreMap.put("foo", 1.45); 646 | valueScoreMap.put("bar", Double.POSITIVE_INFINITY); 647 | valueScoreMap.put("baz", Double.NEGATIVE_INFINITY); 648 | jedis.flushAll(); 649 | setTestConfig("zset-max-ziplist-entries", "0"); 650 | for (Map.Entry e : valueScoreMap.entrySet()) { 651 | jedis.zadd("foo", e.getValue(), e.getKey()); 652 | } 653 | jedis.save(); 654 | restoreConfig("zset-max-ziplist-entries"); 655 | try (RdbParser p = openTestParser()) { 656 | skipToFirstKeyValuePair(p); 657 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 658 | Assert.assertEquals(ValueType.SORTED_SET, kvp.getValueType()); 659 | Map parsedValueScoreMap = new HashMap(); 660 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 661 | parsedValueScoreMap.put(str(i.next()), RdbParser.parseSortedSetScore(i.next())); 662 | } 663 | Assert.assertEquals(valueScoreMap, parsedValueScoreMap); 664 | } 665 | } 666 | } 667 | 668 | @Test 669 | public void sortedSet2() throws Exception { 670 | if (rdbVersion >= 8) { 671 | Map valueScoreMap = new HashMap(); 672 | valueScoreMap.put("foo", 1.45); 673 | valueScoreMap.put("bar", Double.POSITIVE_INFINITY); 674 | valueScoreMap.put("baz", Double.NEGATIVE_INFINITY); 675 | jedis.flushAll(); 676 | setTestConfig("zset-max-ziplist-entries", "0"); 677 | for (Map.Entry e : valueScoreMap.entrySet()) { 678 | jedis.zadd("foo", e.getValue(), e.getKey()); 679 | } 680 | jedis.save(); 681 | restoreConfig("zset-max-ziplist-entries"); 682 | try (RdbParser p = openTestParser()) { 683 | skipToFirstKeyValuePair(p); 684 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 685 | Assert.assertEquals(ValueType.SORTED_SET2, kvp.getValueType()); 686 | Map parsedValueScoreMap = new HashMap(); 687 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 688 | parsedValueScoreMap.put(str(i.next()), RdbParser.parseSortedSet2Score(i.next())); 689 | } 690 | Assert.assertEquals(valueScoreMap, parsedValueScoreMap); 691 | } 692 | } 693 | } 694 | 695 | @Test 696 | public void hash() throws Exception { 697 | String maxEntriesKey = isEarlierThan(redisVersion, "2.6.0") ? "hash-max-zipmap-entries" 698 | : "hash-max-ziplist-entries"; 699 | setTestConfig(maxEntriesKey, "0"); 700 | Map map = new HashMap(); 701 | map.put("one", "loremipsum"); 702 | map.put("two", "2"); 703 | map.put("three", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 704 | jedis.flushAll(); 705 | for (Map.Entry e : map.entrySet()) { 706 | jedis.hset("foo", e.getKey(), e.getValue()); 707 | } 708 | jedis.save(); 709 | restoreConfig(maxEntriesKey); 710 | try (RdbParser p = openTestParser()) { 711 | skipToFirstKeyValuePair(p); 712 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 713 | Assert.assertEquals(ValueType.HASH, kvp.getValueType()); 714 | Map parsedMap = new HashMap(); 715 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 716 | parsedMap.put(str(i.next()), str(i.next())); 717 | } 718 | Assert.assertEquals(map, parsedMap); 719 | } 720 | } 721 | 722 | @Test 723 | public void quickList() throws Exception { 724 | if (rdbVersion >= 7) { 725 | List list = Arrays.asList("loremipsum", // string 726 | "10", // 4 bit integer 727 | "30", // 8 bit integer 728 | "-30", // 8 bit signed integer 729 | "1000", // 16 bit integer 730 | "-1000", // 16 bit signed integer 731 | "300000", // 24 bit integer 732 | "-300000", // 24 bit signed integer 733 | "30000000", // 32 bit integer 734 | "-30000000", // 32 bit signed integer 735 | "9000000000", // 64 bit integer 736 | "-9000000000" // 64 bit signed integer 737 | ); 738 | jedis.flushAll(); 739 | for (String s : list) { 740 | jedis.lpush("foo", s); 741 | } 742 | jedis.save(); 743 | try (RdbParser p = openTestParser()) { 744 | skipToFirstKeyValuePair(p); 745 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 746 | if (rdbVersion >= 10) { 747 | Assert.assertEquals(ValueType.QUICKLIST2, kvp.getValueType()); 748 | } else { 749 | Assert.assertEquals(ValueType.QUICKLIST, kvp.getValueType()); 750 | } 751 | List parsedList = new ArrayList(); 752 | for (byte[] val : kvp.getValues()) { 753 | parsedList.add(str(val)); 754 | } 755 | Collections.reverse(parsedList); 756 | Assert.assertEquals(list, parsedList); 757 | } 758 | } 759 | } 760 | 761 | @Test 762 | public void intSet16Bit() throws Exception { 763 | Set ints = new HashSet(); 764 | Collections.addAll(ints, "1", "-1", "12", "-12"); 765 | jedis.flushAll(); 766 | for (String s : ints) { 767 | jedis.sadd("foo", s); 768 | } 769 | jedis.save(); 770 | try (RdbParser p = openTestParser()) { 771 | skipToFirstKeyValuePair(p); 772 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 773 | Assert.assertEquals(ValueType.INTSET, kvp.getValueType()); 774 | Set parsedInts = new HashSet(); 775 | for (byte[] bs : kvp.getValues()) { 776 | parsedInts.add(str(bs)); 777 | } 778 | Assert.assertEquals(ints, parsedInts); 779 | } 780 | } 781 | 782 | @Test 783 | public void intSet32Bit() throws Exception { 784 | Set ints = new HashSet(); 785 | Collections.addAll(ints, "1", "-1", "30000000", "-30000000"); 786 | jedis.flushAll(); 787 | for (String s : ints) { 788 | jedis.sadd("foo", s); 789 | } 790 | jedis.save(); 791 | try (RdbParser p = openTestParser()) { 792 | skipToFirstKeyValuePair(p); 793 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 794 | Assert.assertEquals(ValueType.INTSET, kvp.getValueType()); 795 | Set parsedInts = new HashSet(); 796 | for (byte[] bs : kvp.getValues()) { 797 | parsedInts.add(str(bs)); 798 | } 799 | Assert.assertEquals(ints, parsedInts); 800 | } 801 | } 802 | 803 | @Test 804 | public void intSet64Bit() throws Exception { 805 | Set ints = new HashSet(); 806 | Collections.addAll(ints, "1", "-1", "9000000000", "-9000000000"); 807 | jedis.flushAll(); 808 | for (String s : ints) { 809 | jedis.sadd("foo", s); 810 | } 811 | jedis.save(); 812 | try (RdbParser p = openTestParser()) { 813 | skipToFirstKeyValuePair(p); 814 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 815 | Assert.assertEquals(ValueType.INTSET, kvp.getValueType()); 816 | Set parsedInts = new HashSet(); 817 | for (byte[] bs : kvp.getValues()) { 818 | parsedInts.add(str(bs)); 819 | } 820 | Assert.assertEquals(ints, parsedInts); 821 | } 822 | } 823 | 824 | private void checkCompactedSortedSet(ValueType expectedType) throws Exception { 825 | Map valueScoreMap = new HashMap(); 826 | valueScoreMap.put("foo", 1.45); 827 | valueScoreMap.put("bar", Double.POSITIVE_INFINITY); 828 | valueScoreMap.put("baz", Double.NEGATIVE_INFINITY); 829 | jedis.flushAll(); 830 | for (Map.Entry e : valueScoreMap.entrySet()) { 831 | jedis.zadd("foo", e.getValue(), e.getKey()); 832 | } 833 | jedis.save(); 834 | try (RdbParser p = openTestParser()) { 835 | skipToFirstKeyValuePair(p); 836 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 837 | Assert.assertEquals(expectedType, kvp.getValueType()); 838 | Map parsedValueScoreMap = new HashMap(); 839 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 840 | parsedValueScoreMap.put(str(i.next()), RdbParser.parseSortedSetScore(i.next())); 841 | } 842 | Assert.assertEquals(valueScoreMap, parsedValueScoreMap); 843 | } 844 | } 845 | 846 | @Test 847 | public void sortedSetAsZipList() throws Exception { 848 | if (rdbVersion <= 9) { 849 | setTestConfig("zset-max-ziplist-entries", "64"); 850 | setTestConfig("zset-max-ziplist-value", "64"); 851 | checkCompactedSortedSet(ValueType.SORTED_SET_AS_ZIPLIST); 852 | restoreConfig("zset-max-ziplist-entries"); 853 | restoreConfig("zset-max-ziplist-value"); 854 | } 855 | } 856 | 857 | @Test 858 | public void sortedSetAsListpack() throws Exception { 859 | if (rdbVersion >= 10) { 860 | setTestConfig("zset-max-listpack-entries", "64"); 861 | setTestConfig("zset-max-listpack-value", "64"); 862 | checkCompactedSortedSet(ValueType.SORTED_SET_AS_LISTPACK); 863 | restoreConfig("zset-max-listpack-entries"); 864 | restoreConfig("zset-max-listpack-value"); 865 | } 866 | } 867 | 868 | @Test 869 | public void hashmapAsZipList() throws Exception { 870 | if (isLaterThan(redisVersion, "2.6.0") && rdbVersion <= 9) { 871 | StringBuffer sb = new StringBuffer(16384); 872 | for (int i = 0; i < 64; ++i) { 873 | sb.append("x"); 874 | } 875 | String mediumString = sb.toString(); 876 | for (int i = 0; i < 256; ++i) { 877 | sb.append(mediumString); 878 | } 879 | String longString = sb.toString(); 880 | Map map = new HashMap(); 881 | map.put("one", "loremipsum"); 882 | map.put("two", "2"); 883 | map.put("three", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 884 | map.put("four", mediumString); 885 | map.put("five", longString); 886 | setTestConfig("hash-max-ziplist-entries", "64"); 887 | setTestConfig("hash-max-ziplist-value", Integer.toString(longString.length())); 888 | jedis.flushAll(); 889 | for (Map.Entry e : map.entrySet()) { 890 | jedis.hset("foo", e.getKey(), e.getValue()); 891 | } 892 | jedis.save(); 893 | restoreConfig("hash-max-ziplist-entries"); 894 | restoreConfig("hash-max-ziplist-value"); 895 | try (RdbParser p = openTestParser()) { 896 | skipToFirstKeyValuePair(p); 897 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 898 | Assert.assertEquals(ValueType.HASHMAP_AS_ZIPLIST, kvp.getValueType()); 899 | Map parsedMap = new HashMap(); 900 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 901 | parsedMap.put(str(i.next()), str(i.next())); 902 | } 903 | Assert.assertEquals(map, parsedMap); 904 | } 905 | } 906 | } 907 | 908 | @Test 909 | public void hashmapAsListpack() throws Exception { 910 | if (rdbVersion >= 10) { 911 | StringBuffer sb = new StringBuffer(16384); 912 | for (int i = 0; i < 64; ++i) { 913 | sb.append("x"); 914 | } 915 | String mediumString = sb.toString(); 916 | for (int i = 0; i < 256; ++i) { 917 | sb.append(mediumString); 918 | } 919 | String longString = sb.toString(); 920 | List list = Arrays.asList("30", // 7 bit integer 921 | "500", // 13 bit signed integer 922 | "-30", // 13 bit signed integer 923 | "16000", // 16 bit integer 924 | "-16000", // 16 bit signed integer 925 | "300000", // 24 bit integer 926 | "-300000", // 24 bit signed integer 927 | "30000000", // 32 bit integer 928 | "-30000000", // 32 bit signed integer 929 | "9000000000", // 64 bit integer 930 | "-9000000000", // 64 bit signed integer 931 | "loremipsum", // 6 bit string 932 | mediumString, // 12 bit string 933 | longString // 32 bit string 934 | ); 935 | Map map = new HashMap(); 936 | for (int i = 0; i < list.size(); ++i) { 937 | map.put(Integer.toString(i), list.get(i)); 938 | } 939 | setTestConfig("hash-max-listpack-value", Integer.toString(longString.length())); 940 | jedis.flushAll(); 941 | for (Map.Entry e : map.entrySet()) { 942 | jedis.hset("foo", e.getKey(), e.getValue()); 943 | } 944 | jedis.save(); 945 | restoreConfig("hash-max-listpack-value"); 946 | try (RdbParser p = openTestParser()) { 947 | skipToFirstKeyValuePair(p); 948 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 949 | Assert.assertEquals(ValueType.HASHMAP_AS_LISTPACK, kvp.getValueType()); 950 | Map parsedMap = new HashMap(); 951 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 952 | parsedMap.put(str(i.next()), str(i.next())); 953 | } 954 | Assert.assertEquals(map, parsedMap); 955 | } 956 | } 957 | } 958 | 959 | @Test 960 | public void zipMap() throws Exception { 961 | if (isEarlierThan(redisVersion, "2.6.0")) { 962 | Map map = new HashMap(); 963 | map.put("one", "loremipsum"); 964 | map.put("two", "2"); 965 | map.put("three", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 966 | jedis.flushAll(); 967 | for (Map.Entry e : map.entrySet()) { 968 | jedis.hset("foo", e.getKey(), e.getValue()); 969 | } 970 | jedis.save(); 971 | try (RdbParser p = openTestParser()) { 972 | skipToFirstKeyValuePair(p); 973 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 974 | Assert.assertEquals(ValueType.ZIPMAP, kvp.getValueType()); 975 | Map parsedMap = new HashMap(); 976 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 977 | parsedMap.put(str(i.next()), str(i.next())); 978 | } 979 | Assert.assertEquals(map, parsedMap); 980 | } 981 | } 982 | } 983 | 984 | @Test 985 | public void setAsListpack() throws Exception { 986 | if (rdbVersion >= 11) { 987 | StringBuffer sb = new StringBuffer(16384); 988 | for (int i = 0; i < 64; ++i) { 989 | sb.append("x"); 990 | } 991 | String mediumString = sb.toString(); 992 | for (int i = 0; i < 256; ++i) { 993 | sb.append(mediumString); 994 | } 995 | String longString = sb.toString(); 996 | List list = Arrays.asList("30", // 7 bit integer 997 | "500", // 13 bit signed integer 998 | "-30", // 13 bit signed integer 999 | "16000", // 16 bit integer 1000 | "-16000", // 16 bit signed integer 1001 | "300000", // 24 bit integer 1002 | "-300000", // 24 bit signed integer 1003 | "30000000", // 32 bit integer 1004 | "-30000000", // 32 bit signed integer 1005 | "9000000000", // 64 bit integer 1006 | "-9000000000", // 64 bit signed integer 1007 | "loremipsum", // 6 bit string 1008 | mediumString, // 12 bit string 1009 | longString // 32 bit string 1010 | ); 1011 | Set set = new HashSet(); 1012 | for (int i = 0; i < list.size(); ++i) { 1013 | set.add(list.get(i)); 1014 | } 1015 | setTestConfig("set-max-listpack-value", Integer.toString(longString.length())); 1016 | jedis.flushAll(); 1017 | for (String e : set) { 1018 | jedis.sadd("foo", e); 1019 | } 1020 | jedis.save(); 1021 | restoreConfig("set-max-listpack-value"); 1022 | try (RdbParser p = openTestParser()) { 1023 | skipToFirstKeyValuePair(p); 1024 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 1025 | Assert.assertEquals(ValueType.SET_AS_LISTPACK, kvp.getValueType()); 1026 | Set parsedSet = new HashSet(); 1027 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 1028 | parsedSet.add(str(i.next())); 1029 | } 1030 | Assert.assertEquals(set, parsedSet); 1031 | } 1032 | } 1033 | } 1034 | 1035 | @Test 1036 | public void hashMetadata() throws Exception { 1037 | if (rdbVersion >= 12) { 1038 | Map map = new HashMap(); 1039 | Map expireMap = new HashMap(); 1040 | map.put("one", "loremipsum"); 1041 | map.put("two", "2"); 1042 | map.put("three", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 1043 | long earliestExpire = 86L; 1044 | long minExpireMS = System.currentTimeMillis() + earliestExpire * 1000; 1045 | expireMap.put("one", minExpireMS + 1); 1046 | expireMap.put("two", 0L); 1047 | expireMap.put("three", minExpireMS + 2000L * 1000 + 1); 1048 | setTestConfig("hash-max-listpack-value", "0"); 1049 | jedis.flushAll(); 1050 | for (Map.Entry e : map.entrySet()) { 1051 | jedis.hset("foo", e.getKey(), e.getValue()); 1052 | long expireTime = expireMap.get(e.getKey()); 1053 | if (expireTime == 0) { 1054 | continue; 1055 | } 1056 | jedis.hexpire("foo", ((expireTime - minExpireMS - 1) / 1000) + earliestExpire, e.getKey()); 1057 | } 1058 | jedis.save(); 1059 | restoreConfig("hash-max-listpack-value"); 1060 | try (RdbParser p = openTestParser()) { 1061 | skipToFirstKeyValuePair(p); 1062 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 1063 | Assert.assertEquals(ValueType.HASHMAP_WITH_METADATA, kvp.getValueType()); 1064 | Map parsedMap = new HashMap(); 1065 | Map parsedExpireMap = new HashMap(); 1066 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 1067 | String key = str(i.next()); 1068 | parsedMap.put(key, str(i.next())); 1069 | parsedExpireMap.put(key, Long.parseLong(str(i.next()))); 1070 | } 1071 | Assert.assertTrue("expected minHashExpireTime: " + minExpireMS 1072 | + ", actual: " + kvp.getMinHashExpireTime(), 1073 | Math.abs(minExpireMS - kvp.getMinHashExpireTime()) 1074 | <= EXPIRATION_TOLERANCE_MS); 1075 | Assert.assertEquals(map, parsedMap); 1076 | for (Map.Entry e : expireMap.entrySet()) { 1077 | String key = e.getKey(); 1078 | long expected = e.getValue(); 1079 | long actual = parsedExpireMap.get(key); 1080 | Assert.assertTrue("key: " + key + ", expected: " + expected 1081 | + ", actual: " + actual, 1082 | Math.abs(expected - actual) <= EXPIRATION_TOLERANCE_MS); 1083 | } 1084 | } 1085 | } 1086 | } 1087 | 1088 | 1089 | @Test 1090 | public void hashmapListpackEx() throws Exception { 1091 | if (rdbVersion >= 12) { 1092 | StringBuffer sb = new StringBuffer(16384); 1093 | for (int i = 0; i < 64; ++i) { 1094 | sb.append("x"); 1095 | } 1096 | String mediumString = sb.toString(); 1097 | for (int i = 0; i < 256; ++i) { 1098 | sb.append(mediumString); 1099 | } 1100 | String longString = sb.toString(); 1101 | List list = Arrays.asList("30", // 7 bit integer 1102 | "500", // 13 bit signed integer 1103 | "-30", // 13 bit signed integer 1104 | "16000", // 16 bit integer 1105 | "-16000", // 16 bit signed integer 1106 | "300000", // 24 bit integer 1107 | "-300000", // 24 bit signed integer 1108 | "30000000", // 32 bit integer 1109 | "-30000000", // 32 bit signed integer 1110 | "9000000000", // 64 bit integer 1111 | "-9000000000", // 64 bit signed integer 1112 | "loremipsum", // 6 bit string 1113 | mediumString, // 12 bit string 1114 | longString // 32 bit string 1115 | ); 1116 | Map map = new HashMap(); 1117 | Map expireMap = new HashMap(); 1118 | for (int i = 0; i < list.size(); ++i) { 1119 | map.put(Integer.toString(i), list.get(i)); 1120 | } 1121 | long earliestExpire = 86L; 1122 | long minExpireMS = System.currentTimeMillis() + earliestExpire * 1000; 1123 | setTestConfig("hash-max-listpack-value", Integer.toString(longString.length())); 1124 | jedis.flushAll(); 1125 | for (Map.Entry e : map.entrySet()) { 1126 | String key = e.getKey(); 1127 | int i = Integer.parseInt(key); 1128 | jedis.hset("foo", key, e.getValue()); 1129 | if (i % 3 == 0) { // leave some keys without expiration 1130 | expireMap.put(key, 0L); 1131 | continue; 1132 | } 1133 | expireMap.put(key, minExpireMS + (i * 1000L) + 1); 1134 | jedis.hexpire("foo", earliestExpire + i, e.getKey()); 1135 | } 1136 | jedis.save(); 1137 | restoreConfig("hash-max-listpack-value"); 1138 | try (RdbParser p = openTestParser()) { 1139 | skipToFirstKeyValuePair(p); 1140 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 1141 | Assert.assertEquals(ValueType.HASHMAP_AS_LISTPACK_EX, kvp.getValueType()); 1142 | Map parsedMap = new HashMap(); 1143 | Map parsedExpireMap = new HashMap(); 1144 | for (Iterator i = kvp.getValues().iterator(); i.hasNext();) { 1145 | String key = str(i.next()); 1146 | parsedMap.put(key, str(i.next())); 1147 | parsedExpireMap.put(key, Long.parseLong(str(i.next()))); 1148 | } 1149 | Assert.assertTrue("expected minHashExpireTime: " + minExpireMS 1150 | + ", actual: " + kvp.getMinHashExpireTime(), 1151 | Math.abs(minExpireMS - kvp.getMinHashExpireTime()) 1152 | <= EXPIRATION_TOLERANCE_MS); 1153 | Assert.assertEquals(map, parsedMap); 1154 | for (Map.Entry e : expireMap.entrySet()) { 1155 | String key = e.getKey(); 1156 | long expected = e.getValue(); 1157 | long actual = parsedExpireMap.get(key); 1158 | Assert.assertTrue("key: " + key + ", expected: " + expected 1159 | + ", actual: " + actual, 1160 | Math.abs(expected - actual) <= EXPIRATION_TOLERANCE_MS); 1161 | } 1162 | } 1163 | } 1164 | } 1165 | 1166 | @Test 1167 | public void memoryPolicyLRU() throws Exception { 1168 | if (rdbVersion >= 9) { 1169 | jedis.flushAll(); 1170 | setTestConfig("maxmemory-policy", "allkeys-lru"); 1171 | jedis.set("foo", "bar"); 1172 | jedis.save(); 1173 | try (RdbParser p = openTestParser()) { 1174 | skipToFirstKeyValuePair(p); 1175 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 1176 | Assert.assertEquals("foo", str(kvp.getKey())); 1177 | Assert.assertEquals("bar", str(kvp.getValues().get(0))); 1178 | Assert.assertEquals(0L, (long) kvp.getIdle()); 1179 | } 1180 | restoreConfig("maxmemory-policy"); 1181 | } 1182 | } 1183 | 1184 | @Test 1185 | public void memoryPolicyLFU() throws Exception { 1186 | if (rdbVersion >= 9) { 1187 | jedis.flushAll(); 1188 | setTestConfig("maxmemory-policy", "allkeys-lfu"); 1189 | jedis.set("foo", "bar"); 1190 | jedis.save(); 1191 | try (RdbParser p = openTestParser()) { 1192 | skipToFirstKeyValuePair(p); 1193 | KeyValuePair kvp = (KeyValuePair) p.readNext(); 1194 | Assert.assertEquals("foo", str(kvp.getKey())); 1195 | Assert.assertEquals("bar", str(kvp.getValues().get(0))); 1196 | Assert.assertEquals(5L, (long) kvp.getFreq()); 1197 | } 1198 | restoreConfig("maxmemory-policy"); 1199 | } 1200 | } 1201 | } 1202 | --------------------------------------------------------------------------------