├── .gitignore ├── LICENSE ├── README.md ├── code ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── garychenc │ │ │ └── filemap │ │ │ ├── AbstractFileMap.java │ │ │ ├── FileMap.java │ │ │ ├── RepeatableKeyFileMap.java │ │ │ ├── UniqueKeyFileMap.java │ │ │ ├── store │ │ │ ├── Store.java │ │ │ ├── StoreEntry.java │ │ │ ├── StoreException.java │ │ │ ├── StoreIsFullException.java │ │ │ ├── StoreReader.java │ │ │ ├── StoreReaderException.java │ │ │ ├── StoreWriter.java │ │ │ ├── StoreWriterException.java │ │ │ ├── VersionConflictedException.java │ │ │ └── filestore │ │ │ │ ├── AddExceededMaxBucketsNumberException.java │ │ │ │ ├── BinaryFileStore.java │ │ │ │ ├── BinaryFileStoreConvertor.java │ │ │ │ ├── BinaryFileStoreKey.java │ │ │ │ ├── BinaryFileStoreValue.java │ │ │ │ ├── ContentStoreBlock.java │ │ │ │ ├── ContentStoreBlockException.java │ │ │ │ ├── ContentStoreBlockLengthExceedException.java │ │ │ │ ├── ContentStoreMaster.java │ │ │ │ ├── ContentStoreMasterException.java │ │ │ │ ├── ContentStoreSlave.java │ │ │ │ ├── ContentStoreSlaveException.java │ │ │ │ ├── ContentStoreSlaveIndex.java │ │ │ │ ├── ContentStoreSlaveStoreFile.java │ │ │ │ ├── ErrorEntryRemovedListener.java │ │ │ │ ├── ExceededMaxAllowedContentStoreSlavesNumber.java │ │ │ │ ├── FileBaseBinaryFileStoreConvertorImpl.java │ │ │ │ ├── FileBaseBinaryFileStoreKeyImpl.java │ │ │ │ ├── FileBaseBinaryFileStoreValueImpl.java │ │ │ │ ├── FileBaseContentStoreBlockImpl.java │ │ │ │ ├── FileBaseContentStoreMaster.java │ │ │ │ ├── FileBaseContentStoreSlave.java │ │ │ │ ├── IndexBucket.java │ │ │ │ ├── IndexGrid.java │ │ │ │ ├── LongStoreKey.java │ │ │ │ ├── MallocContentStoreSpaceException.java │ │ │ │ ├── ManagedStoreFileMetaData.java │ │ │ │ ├── SearchReachedMaxBucketsNumberException.java │ │ │ │ ├── StoreKey.java │ │ │ │ ├── StoreValue.java │ │ │ │ └── StringStoreKey.java │ │ │ └── util │ │ │ ├── Asserts.java │ │ │ ├── ByteArrayObjectInput.java │ │ │ ├── ByteArrayObjectOutput.java │ │ │ ├── DeleteDirectoryContentsFailedException.java │ │ │ ├── HexDumper.java │ │ │ ├── IOHelper.java │ │ │ ├── ObjectCache.java │ │ │ ├── ObjectCacheFactory.java │ │ │ ├── SerializationUtils.java │ │ │ ├── SimpleObjectCacheFactoryImpl.java │ │ │ ├── SimpleObjectCacheImpl.java │ │ │ └── Utils.java │ └── resources │ │ └── for-location.txt │ └── test │ ├── java │ └── com │ │ └── github │ │ └── garychenc │ │ └── filemap │ │ └── impl │ │ └── test │ │ ├── RepeatableKeyFileMapTest.java │ │ └── UniqueKeyFileMapTest.java │ └── resources │ └── for-location.txt └── doc └── img ├── Overview-1.png ├── Overview-2.png └── Sys-Arch.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 简介 2 | 3 | ## FileMap 是一个用 Java 编写的，基于文件的高性能本地 Key - Value 存储库，可以理解为一个将数据存储在文件上的 Map，因此，其可存储巨大的数据量，而且具有持久化的效果。 4 | 5 | FileMap 使用非常简单，创建一个 com.github.garychenc.filemap.RepeatableKeyFileMap 对象或 com.github.garychenc.filemap.UniqueKeyFileMap 对象即可开始使用。创建对象的时候指定数据存储的本地目录即可。 6 | 7 | UniqueKeyFileMap 类确保添加新 Key - Value 的时候，新的 Key 与目前 Map 中所有的 Key 都不重复才能添加成功。 8 | 9 | RepeatableKeyFileMap 类允许添加新 Key - Value 的时候，新的 Key 可与原来 Map 中已经存在的 Key 重复。假如需要 Key 的唯一性，由客户端自己保证。 10 | 11 | UniqueKeyFileMap 类由于每次 add 操作都需要搜索 FileMap，确保即将添加的 key 不存在，所以性能会比 RepeatableKeyFileMap 差一些。而 RepeatableKeyFileMap则每次直接将记录添加到 FileMap 中，不进行判断，性能比 UniqueKeyFileMap 好一些，假如应用能够确保产生的 key 的唯一性，并且即使 key 重复也不造成大的影响，则使用 RepeatableKeyFileMap 性能好一些。 12 | 13 | UniqueKeyFileMap 和 RepeatableKeyFileMap 类都是线程安全的。 14 | 15 | FileMap 采用乐观离线锁对记录进行并发更新管理，所以，每次更新或者移除记录的时候需要提供预期的版本号，假如版本号不正确，说明在此之前记录已经被修改了，更新或移除此记录会抛出 VersionConflictedException，需要重新读出记录，拿到最新的版本号之后再进行更新或移除操作。一般操作模式为 ： 16 | 17 | ``` 18 | 19 | do { 20 | try { 21 | StoreEntry testValue = fileMap.read("TEST-KEY-1"); 22 | fileMap.update((String) testValue.getKey(), "TEST-VALUE-111100-1", testValue.getVersionNumber()); 23 | break; 24 | } catch (VersionConflictedException e) { 25 | continue; 26 | } catch (StoreReaderException | StoreWriterException e) { 27 | e.printStackTrace(); 28 | break; 29 | } 30 | } while (true); 31 | 32 | ``` 33 | 34 | FileMap 使用完毕之后，必须调用其 close() 方法关闭 FileMap，否则可能造成资源泄露。 35 | 36 | FileMap 基本使用方法示例 ： 37 | 38 | ``` 39 | 40 | try (FileMap fileMap = new UniqueKeyFileMap(storeDir.getAbsolutePath(), "Test-1")) { 41 | 42 | fileMap.add("TEST-KEY-1", "TEST-VALUE-1"); 43 | fileMap.add("TEST-KEY-2", "TEST-VALUE-2"); 44 | 45 | System.out.println(fileMap.size()); // Size = 2 46 | 47 | StoreEntry testValue1 = fileMap.read("TEST-KEY-1"); 48 | StoreEntry testValue2 = fileMap.read("TEST-KEY-2"); 49 | 50 | System.out.println(testValue1.getValue()); // Value = "TEST-VALUE-1" 51 | System.out.println(testValue1.getVersionNumber()); // Version = 0 52 | 53 | System.out.println(testValue2.getValue()); // Value = "TEST-VALUE-2" 54 | System.out.println(testValue2.getVersionNumber()); // Version = 0 55 | 56 | fileMap.update("TEST-KEY-2", "TEST-VALUE-2-001", testValue2.getVersionNumber()); 57 | 58 | testValue2 = fileMap.read("TEST-KEY-2"); 59 | System.out.println(testValue2.getValue()); // Value = "TEST-VALUE-2-001" 60 | System.out.println(testValue2.getVersionNumber()); // Version = 1 61 | 62 | fileMap.remove("TEST-KEY-1", testValue1.getVersionNumber()); 63 | fileMap.remove("TEST-KEY-2", testValue2.getVersionNumber()); 64 | } 65 | 66 | ``` 67 | 68 | # 快速入门 69 | 70 | 接下来将会介绍如何将 FileMap 快速使用起来。首先，读者需要先安装 JDK 和 Maven，并且具有一定的 Java 开发知识，具有一个可用的 Java 开发 IDE。 71 | 72 | + 用 GIT clone 该项目的源代码，代码路径：https://github.com/garychenc/file-map.git 。 73 | 74 | + 使用 eclipse 或 idea 等 Java 开发 IDE 将代码以 Maven 项目的形式导入到 IDE 中。代码在 clone 路径的 code 目录中。 75 | 76 | + 在 IDE 中直接运行 test 源码目录的 com.github.garychenc.filemap.impl.test 包中的测试案例，无须做任何配置。 77 | 78 | + RepeatableKeyFileMapTest 测试案例对 RepeatableKeyFileMap 类进行测试。UniqueKeyFileMapTest 测试案例对 UniqueKeyFileMap 类进行测试。 79 | 80 | + testNormalUseCase 测试基本使用 case，testMultiThreadNormalUseCase1，testMultiThreadNormalUseCase2 进行并发使用测试，并且也可作为性能测试案例使用。 81 | 82 | + 通过对 com.github.garychenc.filemap.impl.test 包中的测试案例进行学习，即可掌握 FileMap 的使用方法。由于是作为类库使用，无须任何配置。 83 | 84 | + 使用 IDE 中 Maven 的 install 功能即可将 FileMap 打包为一个 jar 包，在其它项目中使用。 85 | 86 | # 联系方式 87 | 88 | Gary CHEN : email : gary.chen.c@qq.com 89 | 90 | # 系统架构简介 91 | 92 | **系统整体设计思想 ：** 93 | 94 | ![Overview-1](https://github.com/garychenc/file-map/blob/master/doc/img/Overview-1.png "系统整体设计思想") 95 | 96 | **系统设计思想详细描述 ：** 97 | 98 | ![Overview-2](https://github.com/garychenc/file-map/blob/master/doc/img/Overview-2.png "系统设计思想详细描述") 99 | 100 | **系统详细架构设计描述 ：** 101 | 102 | ![Archi-1](https://github.com/garychenc/file-map/blob/master/doc/img/Sys-Arch.png "系统详细架构设计描述") 103 | 104 | + FileMap 接口是系统的入口，其实现类有 RepeatableKeyFileMap 和 UniqueKeyFileMap，两个类大同小异，FileMap 实现包含一个 BinaryFileStore 列表，每个 BinaryFileStore 对象代表一个文件存储集合。 105 | 106 | + BinaryFileStore 实现了上述描述的基于内存的 Map 类型索引，索引的每个 IndexBucket 指向一个 FileBaseContentStoreSlave 对象，该对象管理该 IndexBucket 指向的文件存储位置和该位置存储的数据。 FileBaseContentStoreSlave 对象管理着在哪个 Slave Data Store File 的哪个位置存储着真实的数据，并且提供数据的读取、写入方法。一个 Slave Data Store File 可对应多个 FileBaseContentStoreSlave 对象，整个 FileMap 包含多个 Slave Data Store File，数据分布在这些 Slave Data Store File 上，每个文件最大存储 64M 的数据。 107 | 108 | + FileBaseContentStoreMaster 对所有 FileBaseContentStoreSlave 和 Slave Data Store File 进行管理，每次 BinaryFileStore 需要存储一个新数据的时候，首先需要向 FileBaseContentStoreMaster 申请一个 FileBaseContentStoreSlave，申请的时候 FileBaseContentStoreMaster 就会在一个空闲的 Slave Data Store File 中为新申请的 FileBaseContentStoreSlave 分配存储空间，并且返回文件存储位置给 Slave，接着将 Slave 纳入 FileBaseContentStoreMaster 进行管理。然后，BinaryFileStore 通过新申请的 FileBaseContentStoreSlave 将数据存储到刚刚分配的文件存储空间中。假如，没有文件可以分配需要的空间，则新建一个文件，在该文件上分配所需空间，并且将该文件纳入管理列表，对文件进行管理。由于 BinaryFileStore 的 IndexBucket 索引只存储 FileBaseContentStoreSlave 的编号，所以每次读取数据的时候，首先需要通过 FileBaseContentStoreMaster 根据 Slave 编号获取 FileBaseContentStoreSlave 对象，然后通过 FileBaseContentStoreSlave 读取其管理的文件位置的数据。 109 | 110 | + FileBaseContentStoreMaster 具有无用文件存储块和无用文件垃圾回收功能，当某个 FileBaseContentStoreSlave 被标记为删除之后，该 Slave 和其管理的文件存储块将加入已删除 Slave 列表，该列表按照可存储空间从大到小排序，每次申请新的 Slave 的时候会优先从该列表中拿出存储空间刚好合适的 Slave 重新使用。当某个文件包含的所有 FileBaseContentStoreSlave 都被删除之后，该文件和其包含的 FileBaseContentStoreSlave 都会被从系统中删除，以节省内存和磁盘存储空间。但是文件编号保留，下次需要创建新文件的时候，优先使用该文件编号生产新文件的文件名。 111 | 112 | + 对数据进行更新的时候，首先按照数据读取方法的步骤搜索到需要的 FileBaseContentStoreSlave ，看看该 FileBaseContentStoreSlave 的存储空间是否能够容纳下更新后的数据，假如可以，则直接将新的数据写入到该 FileBaseContentStoreSlave 所指向的文件存储块中，假如不能容纳，则向 FileBaseContentStoreMaster 申请新的 FileBaseContentStoreSlave ，然后将数据写入到该新申请的 Slave 中，并且更新内存索引和索引文件中对应的数据。 113 | 114 | + 在数据插入到 FileMap 中的时候，从 FileBaseContentStoreMaster 中申请 FileBaseContentStoreSlave 时就会按照需存储数据实际大小的 2 倍申请存储空间，以使后续更新数据的时候可以在原空间上写入新的数据，除非新数据大小超过了原来数据大小的两倍才需要申请新的 FileBaseContentStoreSlave ，这样就避免了更新数据时，需频繁申请新的 FileBaseContentStoreSlave 。 115 | 116 | + BinaryFileStore 中的内存索引会按照一定的数据结构存储在索引存储文件中，索引存储文件的文件名后缀为 ： meta.db.map，FileBaseContentStoreMaster 管理的 Slave 和 Slave Data Store File 也会按照一定的数据结构存储在 Slave 管理文件和 Slave Data Store File 管理文件中，管理文件的后缀名分别为 ： slaves.management 和 store-files.management。 FileMap 第一次创建的时候会根据可存储数据量参数的大小生成索引存储文件、Slave 管理文件和 Slave Data Store File 管理文件，所以，第一次创建的时候速度会有点慢，可存储数据量参数设置越大，创建越慢，默认可存储数据量参数的大小为 50 万，创建时间为几秒。从第二次开始，创建 FileMap 只是加载索引存储文件、Slave 管理文件和 Slave Data Store File 管理文件，速度会快很多。可存储数据量为 50 万的这些文件总的加载时间为 200 毫秒左右。 FileMap 加载索引存储文件、Slave 管理文件和 Slave Data Store File 管理文件之后，将创建文件中存储的内存索引和 Slave、Slave Data Store File等数据结构。 117 | 118 | + Slave Data Store File 文件名的后缀为 slave.data，文件名最后的号码为文件编号，每个文件最大存储 64M 的数据。当文件大于等于 64M，新申请 FileBaseContentStoreSlave 时，将会新创建 Slave Data Store File， 用于存储新 Slave 的数据，新创建的文件编号将在原来最后一个文件编号的基础上加一。 119 | -------------------------------------------------------------------------------- /code/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.github.garychenc.filemap 7 | file-map 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | file-map 12 | http://maven.apache.org 13 | 14 | 15 | UTF-8 16 | 1.8 17 | 1.8 18 | 1.6.1 19 | 1.1.7 20 | 4.10 21 | 22 | 23 | 24 | 25 | org.slf4j 26 | slf4j-api 27 | ${version.slf4j} 28 | jar 29 | 30 | 31 | 32 | ch.qos.logback 33 | logback-core 34 | ${version.logback} 35 | 36 | 37 | 38 | ch.qos.logback 39 | logback-classic 40 | ${version.logback} 41 | 42 | 43 | 44 | junit 45 | junit 46 | ${version.junit} 47 | test 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-compiler-plugin 56 | 3.5.1 57 | 58 | 1.8 59 | 1.8 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/RepeatableKeyFileMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap; 18 | 19 | import com.github.garychenc.filemap.store.StoreIsFullException; 20 | import com.github.garychenc.filemap.store.StoreReaderException; 21 | import com.github.garychenc.filemap.store.StoreWriterException; 22 | import com.github.garychenc.filemap.store.filestore.BinaryFileStore; 23 | 24 | import java.io.Externalizable; 25 | import java.io.Serializable; 26 | 27 | /** 28 | * RepeatableKeyFileMap 类为 FileMap 的一个实现，该类的 add 方法允许添加新 Key - Value 的时候，新的 Key 可与原来 Map 中已经存在的 Key 重复。假如需要 Key 的唯一性，由客户端自己保证。 29 | * 30 | * @author Gary CHEN 31 | * 32 | */ 33 | public class RepeatableKeyFileMap extends AbstractFileMap { 34 | 35 | /** 36 | * 创建一个 RepeatableKeyFileMap 对象。创建 RepeatableKeyFileMap 对象失败时抛出运行时异常。 37 | * 38 | * @param storeDirPath FileMap 数据存储在文件上时的存储目录。 39 | * @param storeName FileMap 的存储名，存储名用于在同个存储目录中有多个 FileMap 存储时，区分不同的 FileMap 。 40 | * 假如存储目录相同，则每个 FileMap 实例的存储名都必须不一样。两个不同的 FileMap 实例绝对不能使用相同的存储目录和存储名。 41 | * 否则存储的数据可能会互相干扰，出现不可预知的错误。 42 | */ 43 | public RepeatableKeyFileMap(String storeDirPath, String storeName) { 44 | super(storeDirPath, storeName); 45 | } 46 | 47 | /** 48 | * 创建一个 RepeatableKeyFileMap 对象。创建 RepeatableKeyFileMap 对象失败时抛出运行时异常。 49 | * 50 | * @param storeDirPath FileMap 数据存储在文件上时的存储目录。 51 | * @param storeName FileMap 的存储名，存储名用于在同个存储目录中有多个 FileMap 存储时，区分不同的 FileMap 。 52 | * 假如存储目录相同，则每个 FileMap 实例的存储名都必须不一样。两个不同的 FileMap 实例绝对不能使用相同的存储目录和存储名。 53 | * 否则存储的数据可能会互相干扰，出现不可预知的错误。 54 | * @param isFlushFileCacheEveryOperation FileMap 写数据的时候默认先写在操作系统的文件缓存中，然后再刷新文件缓存的数据到磁盘上，该参数 55 | * 控制是否每次对 FileMap 进行 add，update，remove 操作之后都刷新文件缓存的数据到磁盘上。 56 | * true 为每次都刷新，false 则等待 FileMap close 的时候再刷新。 57 | */ 58 | public RepeatableKeyFileMap(String storeDirPath, String storeName, boolean isFlushFileCacheEveryOperation) { 59 | super(storeDirPath, storeName, isFlushFileCacheEveryOperation); 60 | } 61 | 62 | /** 63 | * 创建一个 RepeatableKeyFileMap 对象。创建 RepeatableKeyFileMap 对象失败时抛出运行时异常。 64 | * 65 | * @param storeDirPath FileMap 数据存储在文件上时的存储目录。 66 | * @param storeName FileMap 的存储名，存储名用于在同个存储目录中有多个 FileMap 存储时，区分不同的 FileMap 。 67 | * 假如存储目录相同，则每个 FileMap 实例的存储名都必须不一样。两个不同的 FileMap 实例绝对不能使用相同的存储目录和存储名。 68 | * 否则存储的数据可能会互相干扰，出现不可预知的错误。 69 | * @param isFlushFileCacheEveryOperation FileMap 写数据的时候默认先写在操作系统的文件缓存中，然后再刷新文件缓存的数据到磁盘上，该参数 70 | * 控制是否每次对 FileMap 进行 add，update，remove 操作之后都刷新文件缓存的数据到磁盘上。 71 | * true 为每次都刷新，false 则等待 FileMap close 的时候再刷新。 72 | * @param isStrictStorage 由于上述写数据的特性，所以，当 FileMap 被不正常关闭的时候，有可能出现某些数据丢失、损坏，而出现无法再次读取的情况。 73 | * 该参数控制当出现数据无法读取的时候，是抛出异常，还是忽略该数据，返回 null，并且将该数据从 FileMap 中移除。 74 | * ture 抛出异常，false 忽略该数据。 75 | */ 76 | public RepeatableKeyFileMap(String storeDirPath, String storeName, boolean isFlushFileCacheEveryOperation, boolean isStrictStorage) { 77 | super(storeDirPath, storeName, isFlushFileCacheEveryOperation, isStrictStorage); 78 | } 79 | 80 | /** 81 | * 创建一个 RepeatableKeyFileMap 对象。创建 RepeatableKeyFileMap 对象失败时抛出运行时异常。 82 | * 83 | * @param storeDirPath FileMap 数据存储在文件上时的存储目录。 84 | * @param storeName FileMap 的存储名，存储名用于在同个存储目录中有多个 FileMap 存储时，区分不同的 FileMap 。 85 | * 假如存储目录相同，则每个 FileMap 实例的存储名都必须不一样。两个不同的 FileMap 实例绝对不能使用相同的存储目录和存储名。 86 | * 否则存储的数据可能会互相干扰，出现不可预知的错误。 87 | * @param isFlushFileCacheEveryOperation FileMap 写数据的时候默认先写在操作系统的文件缓存中，然后再刷新文件缓存的数据到磁盘上，该参数 88 | * 控制是否每次对 FileMap 进行 add，update，remove 操作之后都刷新文件缓存的数据到磁盘上。 89 | * true 为每次都刷新，false 则等待 FileMap close 的时候再刷新。 90 | * @param isStrictStorage 由于上述写数据的特性，所以，当 FileMap 被不正常关闭的时候，有可能出现某些数据丢失、损坏，而出现无法再次读取的情况。 91 | * 该参数控制当出现数据无法读取的时候，是抛出异常，还是忽略该数据，返回 null，并且将该数据从 FileMap 中移除。 92 | * ture 抛出异常，false 忽略该数据。 93 | * @param aFileMapStoreMaxSize FileMap 内部采用多个文件存储数据，该参数控制当一个文件存储的记录条数到达多少时创建新的文件存储新的数据。 94 | */ 95 | public RepeatableKeyFileMap(String storeDirPath, String storeName, boolean isFlushFileCacheEveryOperation, boolean isStrictStorage, int aFileMapStoreMaxSize) { 96 | super(storeDirPath, storeName, isFlushFileCacheEveryOperation, isStrictStorage, aFileMapStoreMaxSize); 97 | } 98 | 99 | @Override 100 | public boolean add(String key, byte[] value) throws StoreWriterException { 101 | while (true) { 102 | BinaryFileStore spaceAvailableMapStore = null; 103 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 104 | try { 105 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 106 | spaceAvailableMapStore = fileMapStore; 107 | break; 108 | } 109 | } catch (StoreReaderException ex) { 110 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 111 | } 112 | } 113 | 114 | if (spaceAvailableMapStore != null) { 115 | try { 116 | spaceAvailableMapStore.add(key, value); 117 | return true; 118 | } catch (StoreIsFullException e) { 119 | // continue; 120 | } 121 | } else { 122 | spaceAvailableMapStore = extendNewFileMapStore(); 123 | 124 | try { 125 | spaceAvailableMapStore.add(key, value); 126 | return true; 127 | } catch (StoreIsFullException e) { 128 | // continue; 129 | } 130 | } 131 | } 132 | } 133 | 134 | @Override 135 | public boolean add(long key, byte[] value) throws StoreWriterException { 136 | while (true) { 137 | BinaryFileStore spaceAvailableMapStore = null; 138 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 139 | try { 140 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 141 | spaceAvailableMapStore = fileMapStore; 142 | break; 143 | } 144 | } catch (StoreReaderException ex) { 145 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 146 | } 147 | } 148 | 149 | if (spaceAvailableMapStore != null) { 150 | try { 151 | spaceAvailableMapStore.add(key, value); 152 | return true; 153 | } catch (StoreIsFullException e) { 154 | // continue; 155 | } 156 | } else { 157 | spaceAvailableMapStore = extendNewFileMapStore(); 158 | 159 | try { 160 | spaceAvailableMapStore.add(key, value); 161 | return true; 162 | } catch (StoreIsFullException e) { 163 | // continue; 164 | } 165 | } 166 | } 167 | } 168 | 169 | @Override 170 | public boolean add(String key, String value) throws StoreWriterException { 171 | while (true) { 172 | BinaryFileStore spaceAvailableMapStore = null; 173 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 174 | try { 175 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 176 | spaceAvailableMapStore = fileMapStore; 177 | break; 178 | } 179 | } catch (StoreReaderException ex) { 180 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 181 | } 182 | } 183 | 184 | if (spaceAvailableMapStore != null) { 185 | try { 186 | spaceAvailableMapStore.add(key, value); 187 | return true; 188 | } catch (StoreIsFullException e) { 189 | // continue; 190 | } 191 | } else { 192 | spaceAvailableMapStore = extendNewFileMapStore(); 193 | 194 | try { 195 | spaceAvailableMapStore.add(key, value); 196 | return true; 197 | } catch (StoreIsFullException e) { 198 | // continue; 199 | } 200 | } 201 | } 202 | } 203 | 204 | @Override 205 | public boolean add(long key, String value) throws StoreWriterException { 206 | while (true) { 207 | BinaryFileStore spaceAvailableMapStore = null; 208 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 209 | try { 210 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 211 | spaceAvailableMapStore = fileMapStore; 212 | break; 213 | } 214 | } catch (StoreReaderException ex) { 215 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 216 | } 217 | } 218 | 219 | if (spaceAvailableMapStore != null) { 220 | try { 221 | spaceAvailableMapStore.add(key, value); 222 | return true; 223 | } catch (StoreIsFullException e) { 224 | // continue; 225 | } 226 | } else { 227 | spaceAvailableMapStore = extendNewFileMapStore(); 228 | 229 | try { 230 | spaceAvailableMapStore.add(key, value); 231 | return true; 232 | } catch (StoreIsFullException e) { 233 | // continue; 234 | } 235 | } 236 | } 237 | } 238 | 239 | @Override 240 | public boolean add(String key, Externalizable value) throws StoreWriterException { 241 | while (true) { 242 | BinaryFileStore spaceAvailableMapStore = null; 243 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 244 | try { 245 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 246 | spaceAvailableMapStore = fileMapStore; 247 | break; 248 | } 249 | } catch (StoreReaderException ex) { 250 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 251 | } 252 | } 253 | 254 | if (spaceAvailableMapStore != null) { 255 | try { 256 | spaceAvailableMapStore.add(key, value); 257 | return true; 258 | } catch (StoreIsFullException e) { 259 | // continue; 260 | } 261 | } else { 262 | spaceAvailableMapStore = extendNewFileMapStore(); 263 | 264 | try { 265 | spaceAvailableMapStore.add(key, value); 266 | return true; 267 | } catch (StoreIsFullException e) { 268 | // continue; 269 | } 270 | } 271 | } 272 | } 273 | 274 | @Override 275 | public boolean add(long key, Externalizable value) throws StoreWriterException { 276 | while (true) { 277 | BinaryFileStore spaceAvailableMapStore = null; 278 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 279 | try { 280 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 281 | spaceAvailableMapStore = fileMapStore; 282 | break; 283 | } 284 | } catch (StoreReaderException ex) { 285 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 286 | } 287 | } 288 | 289 | if (spaceAvailableMapStore != null) { 290 | try { 291 | spaceAvailableMapStore.add(key, value); 292 | return true; 293 | } catch (StoreIsFullException e) { 294 | // continue; 295 | } 296 | } else { 297 | spaceAvailableMapStore = extendNewFileMapStore(); 298 | 299 | try { 300 | spaceAvailableMapStore.add(key, value); 301 | return true; 302 | } catch (StoreIsFullException e) { 303 | // continue; 304 | } 305 | } 306 | } 307 | } 308 | 309 | @Override 310 | public boolean add(String key, Serializable value) throws StoreWriterException { 311 | while (true) { 312 | BinaryFileStore spaceAvailableMapStore = null; 313 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 314 | try { 315 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 316 | spaceAvailableMapStore = fileMapStore; 317 | break; 318 | } 319 | } catch (StoreReaderException ex) { 320 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 321 | } 322 | } 323 | 324 | if (spaceAvailableMapStore != null) { 325 | try { 326 | spaceAvailableMapStore.add(key, value); 327 | return true; 328 | } catch (StoreIsFullException e) { 329 | // continue; 330 | } 331 | } else { 332 | spaceAvailableMapStore = extendNewFileMapStore(); 333 | 334 | try { 335 | spaceAvailableMapStore.add(key, value); 336 | return true; 337 | } catch (StoreIsFullException e) { 338 | // continue; 339 | } 340 | } 341 | } 342 | } 343 | 344 | @Override 345 | public boolean add(long key, Serializable value) throws StoreWriterException { 346 | while (true) { 347 | BinaryFileStore spaceAvailableMapStore = null; 348 | for (BinaryFileStore fileMapStore : this.fileMapStores) { 349 | try { 350 | if (spaceAvailableMapStore == null && fileMapStore.size() < aFileMapStoreMaxSize) { 351 | spaceAvailableMapStore = fileMapStore; 352 | break; 353 | } 354 | } catch (StoreReaderException ex) { 355 | throw new StoreWriterException("Check is key exist failed. Key : " + key, ex); 356 | } 357 | } 358 | 359 | if (spaceAvailableMapStore != null) { 360 | try { 361 | spaceAvailableMapStore.add(key, value); 362 | return true; 363 | } catch (StoreIsFullException e) { 364 | // continue; 365 | } 366 | } else { 367 | spaceAvailableMapStore = extendNewFileMapStore(); 368 | 369 | try { 370 | spaceAvailableMapStore.add(key, value); 371 | return true; 372 | } catch (StoreIsFullException e) { 373 | // continue; 374 | } 375 | } 376 | } 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/Store.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface Store { 25 | 26 | public void close() throws StoreException; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 封装了值、版本号和 key 的搜索结果对象，versionNumber 版本号，value 存储的值，key 用于读取值的关键字。 21 | * 22 | * @author Gary CHEN 23 | * 24 | */ 25 | public class StoreEntry { 26 | 27 | private final long versionNumber; 28 | private final Object value; 29 | private final Object key; 30 | 31 | public StoreEntry(long versionNumber, Object value, Object key) { 32 | this.versionNumber = versionNumber; 33 | this.value = value; 34 | this.key = key; 35 | } 36 | 37 | public long getVersionNumber() { 38 | return versionNumber; 39 | } 40 | 41 | public Object getValue() { 42 | return value; 43 | } 44 | 45 | public Object getKey() { 46 | return key; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "StoreEntry [versionNumber=" + versionNumber + ", value=" + value + ", key=" + key + "]"; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | final int prime = 31; 57 | int result = 1; 58 | result = prime * result + ((value == null) ? 0 : value.hashCode()); 59 | result = prime * result + (int) (versionNumber ^ (versionNumber >>> 32)); 60 | return result; 61 | } 62 | 63 | @Override 64 | public boolean equals(Object obj) { 65 | if (this == obj) { 66 | return true; 67 | } 68 | if (obj == null) { 69 | return false; 70 | } 71 | if (!(obj instanceof StoreEntry)) { 72 | return false; 73 | } 74 | StoreEntry other = (StoreEntry) obj; 75 | if (value == null) { 76 | if (other.value != null) { 77 | return false; 78 | } 79 | } else if (!value.equals(other.value)) { 80 | return false; 81 | } 82 | if (versionNumber != other.versionNumber) { 83 | return false; 84 | } 85 | return true; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class StoreException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = -4324407847255959555L; 30 | 31 | /** 32 | * 33 | */ 34 | public StoreException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public StoreException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public StoreException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public StoreException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreIsFullException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class StoreIsFullException extends Exception { 25 | 26 | public StoreIsFullException() { 27 | } 28 | 29 | public StoreIsFullException(String message) { 30 | super(message); 31 | } 32 | 33 | public StoreIsFullException(String message, Throwable cause) { 34 | super(message, cause); 35 | } 36 | 37 | public StoreIsFullException(Throwable cause) { 38 | super(cause); 39 | } 40 | 41 | public StoreIsFullException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { 42 | super(message, cause, enableSuppression, writableStackTrace); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | import java.util.List; 20 | import java.util.Set; 21 | 22 | /** 23 | * 24 | * @author Gary CHEN 25 | * 26 | */ 27 | public interface StoreReader extends Store { 28 | 29 | public StoreEntry read(String key) throws StoreReaderException; 30 | public StoreEntry read(long key) throws StoreReaderException; 31 | 32 | public StoreEntry read(String key, boolean removeWhenEntryCorrupt) throws StoreReaderException; 33 | public StoreEntry read(long key, boolean removeWhenEntryCorrupt) throws StoreReaderException; 34 | 35 | public Set keySet() throws StoreReaderException; 36 | public List values() throws StoreReaderException; 37 | 38 | public Set keySet(boolean ignoreErrorEntry) throws StoreReaderException; 39 | public List values(boolean ignoreErrorEntry) throws StoreReaderException; 40 | 41 | public long size() throws StoreReaderException; 42 | 43 | } 44 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreReaderException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class StoreReaderException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = -6776407438597535467L; 30 | 31 | /** 32 | * 33 | */ 34 | public StoreReaderException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public StoreReaderException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public StoreReaderException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public StoreReaderException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | import java.io.Externalizable; 20 | import java.io.Serializable; 21 | 22 | /** 23 | * 24 | * @author Gary CHEN 25 | * 26 | */ 27 | public interface StoreWriter extends Store { 28 | 29 | public boolean add(String key, byte[] value) throws StoreIsFullException, StoreWriterException; 30 | public boolean add(long key, byte[] value) throws StoreIsFullException, StoreWriterException; 31 | 32 | public boolean add(String key, String value) throws StoreIsFullException, StoreWriterException; 33 | public boolean add(long key, String value) throws StoreIsFullException, StoreWriterException; 34 | 35 | public boolean add(String key, Externalizable value) throws StoreIsFullException, StoreWriterException; 36 | public boolean add(long key, Externalizable value) throws StoreIsFullException, StoreWriterException; 37 | 38 | public boolean add(String key, Serializable value) throws StoreIsFullException, StoreWriterException; 39 | public boolean add(long key, Serializable value) throws StoreIsFullException, StoreWriterException; 40 | 41 | public boolean remove(String key, long versionNumber) throws StoreWriterException, VersionConflictedException; 42 | public boolean remove(long key, long versionNumber) throws StoreWriterException, VersionConflictedException; 43 | 44 | public boolean update(String key, byte[] value, long versionNumber) throws StoreWriterException, VersionConflictedException; 45 | public boolean update(long key, byte[] value, long versionNumber) throws StoreWriterException, VersionConflictedException; 46 | 47 | public boolean update(String key, String value, long versionNumber) throws StoreWriterException, VersionConflictedException; 48 | public boolean update(long key, String value, long versionNumber) throws StoreWriterException, VersionConflictedException; 49 | 50 | public boolean update(String key, Externalizable value, long versionNumber) throws StoreWriterException, VersionConflictedException; 51 | public boolean update(long key, Externalizable value, long versionNumber) throws StoreWriterException, VersionConflictedException; 52 | 53 | public boolean update(String key, Serializable value, long versionNumber) throws StoreWriterException, VersionConflictedException; 54 | public boolean update(long key, Serializable value, long versionNumber) throws StoreWriterException, VersionConflictedException; 55 | 56 | } 57 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/StoreWriterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class StoreWriterException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 5636763335871264539L; 30 | 31 | /** 32 | * 33 | */ 34 | public StoreWriterException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public StoreWriterException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public StoreWriterException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public StoreWriterException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/VersionConflictedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class VersionConflictedException extends Exception { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | private final long providedVersionNumber; 29 | private final long expectedVersionNumber; 30 | 31 | /** 32 | * 33 | */ 34 | public VersionConflictedException(long providedVersionNumber, long expectedVersionNumber) { 35 | this.providedVersionNumber = providedVersionNumber; 36 | this.expectedVersionNumber = expectedVersionNumber; 37 | } 38 | 39 | /** 40 | * @param message 41 | */ 42 | public VersionConflictedException(String message, long providedVersionNumber, long expectedVersionNumber) { 43 | super(message); 44 | this.providedVersionNumber = providedVersionNumber; 45 | this.expectedVersionNumber = expectedVersionNumber; 46 | } 47 | 48 | /** 49 | * @param cause 50 | */ 51 | public VersionConflictedException(Throwable cause, long providedVersionNumber, long expectedVersionNumber) { 52 | super(cause); 53 | this.providedVersionNumber = providedVersionNumber; 54 | this.expectedVersionNumber = expectedVersionNumber; 55 | } 56 | 57 | /** 58 | * @param message 59 | * @param cause 60 | */ 61 | public VersionConflictedException(String message, Throwable cause, long providedVersionNumber, long expectedVersionNumber) { 62 | super(message, cause); 63 | this.providedVersionNumber = providedVersionNumber; 64 | this.expectedVersionNumber = expectedVersionNumber; 65 | } 66 | 67 | public long getProvidedVersionNumber() { 68 | return providedVersionNumber; 69 | } 70 | 71 | public long getExpectedVersionNumber() { 72 | return expectedVersionNumber; 73 | } 74 | 75 | 76 | } 77 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/AddExceededMaxBucketsNumberException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class AddExceededMaxBucketsNumberException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 8517177813255790400L; 30 | 31 | /** 32 | * 33 | */ 34 | public AddExceededMaxBucketsNumberException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public AddExceededMaxBucketsNumberException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public AddExceededMaxBucketsNumberException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public AddExceededMaxBucketsNumberException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/BinaryFileStoreConvertor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface BinaryFileStoreConvertor { 25 | 26 | public ContentStoreBlock toStoreBlock(BinaryFileStoreValue storeValue) throws ContentStoreBlockException; 27 | public BinaryFileStoreValue fromStoreBlock(ContentStoreBlock block) throws ContentStoreBlockException; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/BinaryFileStoreKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface BinaryFileStoreKey extends StoreKey { 25 | 26 | public void setHashValue(int hash); 27 | public int getHashValue(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/BinaryFileStoreValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface BinaryFileStoreValue extends StoreValue { 25 | 26 | public void setVersionNumber(long versionNumber); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreBlock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface ContentStoreBlock { 25 | 26 | public long getContentLength() throws ContentStoreBlockException; 27 | public byte[] toBytes() throws ContentStoreBlockException; 28 | public void fromBytes(byte[] bytes) throws ContentStoreBlockException; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreBlockException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class ContentStoreBlockException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 6432512958342577481L; 30 | 31 | /** 32 | * 33 | */ 34 | public ContentStoreBlockException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public ContentStoreBlockException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public ContentStoreBlockException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public ContentStoreBlockException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreBlockLengthExceedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class ContentStoreBlockLengthExceedException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 734517084849464800L; 30 | 31 | /** 32 | * 33 | */ 34 | public ContentStoreBlockLengthExceedException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public ContentStoreBlockLengthExceedException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public ContentStoreBlockLengthExceedException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public ContentStoreBlockLengthExceedException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreMaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface ContentStoreMaster { 25 | 26 | public ContentStoreSlave malloc(long mallocSize) throws MallocContentStoreSpaceException; 27 | public ContentStoreSlave retrieveSlave(long slaveNumber) throws ContentStoreMasterException; 28 | public void close() throws ContentStoreMasterException; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreMasterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class ContentStoreMasterException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 1963693002658535709L; 30 | 31 | /** 32 | * 33 | */ 34 | public ContentStoreMasterException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public ContentStoreMasterException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public ContentStoreMasterException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public ContentStoreMasterException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreSlave.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface ContentStoreSlave { 25 | 26 | public long getSlaveNumber(); 27 | public void store(ContentStoreBlock block) throws ContentStoreSlaveException, ContentStoreBlockLengthExceedException; 28 | public ContentStoreBlock read() throws ContentStoreSlaveException; 29 | public void delete() throws ContentStoreSlaveException; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreSlaveException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class ContentStoreSlaveException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 3566275701132261151L; 30 | 31 | /** 32 | * 33 | */ 34 | public ContentStoreSlaveException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public ContentStoreSlaveException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public ContentStoreSlaveException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public ContentStoreSlaveException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreSlaveIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import static com.github.garychenc.filemap.util.Asserts.longIsPositive; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.IOException; 24 | import java.util.concurrent.atomic.AtomicBoolean; 25 | import java.util.concurrent.atomic.AtomicInteger; 26 | import java.util.concurrent.atomic.AtomicLong; 27 | 28 | import com.github.garychenc.filemap.util.IOHelper; 29 | import com.github.garychenc.filemap.util.Utils; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * 35 | * @author Gary CHEN 36 | * 37 | */ 38 | public class ContentStoreSlaveIndex implements Comparable { 39 | 40 | private final static Logger LOGGER = LoggerFactory.getLogger(ContentStoreSlaveIndex.class); 41 | 42 | public final static int STORE_SLAVE_INDEX_BYTES_LENGTH = 21; 43 | public final static long NULL_STORE_SIZE = -1; 44 | public final static int NULL_FILE_NUMBER = -1; 45 | public final static long NULL_STORE_START_ADDRESS = -1; 46 | public final static long TOTALLY_REMOVED_STORE_SIZE = -21; 47 | public final static byte[] NULL_STORE_SLAVE_INDEX_BYTES; 48 | 49 | static { 50 | ByteArrayOutputStream outputstream = null; 51 | try { 52 | outputstream = new ByteArrayOutputStream(); 53 | IOHelper.writeBoolean(outputstream, false); 54 | IOHelper.write64bit(outputstream, NULL_STORE_SIZE); 55 | IOHelper.write32bit(outputstream, NULL_FILE_NUMBER); 56 | IOHelper.write64bit(outputstream, NULL_STORE_START_ADDRESS); 57 | outputstream.flush(); 58 | NULL_STORE_SLAVE_INDEX_BYTES = outputstream.toByteArray(); 59 | } catch (Throwable ex) { 60 | throw new IllegalStateException("Write null store slave index bytes failed.", ex); 61 | } finally { 62 | if (outputstream != null) { 63 | try { outputstream.close(); } catch (IOException exce) { } 64 | } 65 | } 66 | } 67 | 68 | //cache line padding 1 69 | private long p1, p2, p3, p4, p5, p6, p7, p8; 70 | 71 | private volatile boolean removed; 72 | private volatile long storeSize; 73 | private volatile int fileNumber; 74 | private volatile long storeStartAddress; 75 | private volatile long slaveNumber; 76 | 77 | //cache line padding 2 78 | private long p9, p10, p11, p12, p13, p14, p15, p16; 79 | 80 | public ContentStoreSlaveIndex(boolean removed, long storeSize, long slaveNumber, int fileNumber, long storeStartAddress) { 81 | longIsPositive("slaveNumber", slaveNumber); 82 | if (storeSize < 0 && storeSize != TOTALLY_REMOVED_STORE_SIZE) { 83 | throw new IllegalArgumentException("storeSize error! storeSize : " + storeSize); 84 | } 85 | 86 | this.removed = removed; 87 | this.storeSize = storeSize; 88 | this.fileNumber = fileNumber; 89 | this.storeStartAddress = storeStartAddress; 90 | this.slaveNumber = slaveNumber; 91 | } 92 | 93 | public boolean isRemoved() { 94 | return removed; 95 | } 96 | 97 | public void setRemoved(boolean removed) { 98 | this.removed = removed; 99 | } 100 | 101 | public long getStoreSize() { 102 | return storeSize; 103 | } 104 | 105 | public void setStoreSize(long storeSize) { 106 | if (storeSize < 0 && storeSize != TOTALLY_REMOVED_STORE_SIZE) { 107 | throw new IllegalArgumentException("storeSize error! storeSize : " + storeSize); 108 | } 109 | 110 | this.storeSize = storeSize; 111 | } 112 | 113 | public long getStoreStartAddress() { 114 | return storeStartAddress; 115 | } 116 | 117 | public void setStoreStartAddress(long storeStartAddress) { 118 | this.storeStartAddress = storeStartAddress; 119 | } 120 | 121 | public int getFileNumber() { 122 | return fileNumber; 123 | } 124 | 125 | public void setFileNumber(int fileNumber) { 126 | this.fileNumber = fileNumber; 127 | } 128 | 129 | public long getSlaveNumber() { 130 | return slaveNumber; 131 | } 132 | 133 | public long getP1() { 134 | return p1; 135 | } 136 | 137 | public long getP2() { 138 | return p2; 139 | } 140 | 141 | public long getP3() { 142 | return p3; 143 | } 144 | 145 | public long getP4() { 146 | return p4; 147 | } 148 | 149 | public long getP5() { 150 | return p5; 151 | } 152 | 153 | public long getP6() { 154 | return p6; 155 | } 156 | 157 | public long getP7() { 158 | return p7; 159 | } 160 | 161 | public long getP8() { 162 | return p8; 163 | } 164 | 165 | public long getP9() { 166 | return p9; 167 | } 168 | 169 | public long getP10() { 170 | return p10; 171 | } 172 | 173 | public long getP11() { 174 | return p11; 175 | } 176 | 177 | public long getP12() { 178 | return p12; 179 | } 180 | 181 | public long getP13() { 182 | return p13; 183 | } 184 | 185 | public long getP14() { 186 | return p14; 187 | } 188 | 189 | public long getP15() { 190 | return p15; 191 | } 192 | 193 | public long getP16() { 194 | return p16; 195 | } 196 | 197 | @Override 198 | public int compareTo(ContentStoreSlaveIndex another) { 199 | if (this.getStoreSize() > another.getStoreSize()) { 200 | return 1; 201 | } else if (this.getStoreSize() < another.getStoreSize()) { 202 | return -1; 203 | } else { 204 | return 0; 205 | } 206 | } 207 | 208 | public static ContentStoreSlaveIndex fromBytes(byte[] bytes, long slaveNumber, boolean ignoreErrorEntry) throws IOException { 209 | if (ignoreErrorEntry) { 210 | if (bytes == null || bytes.length != STORE_SLAVE_INDEX_BYTES_LENGTH) { 211 | return new ContentStoreSlaveIndex(true, TOTALLY_REMOVED_STORE_SIZE, slaveNumber, NULL_FILE_NUMBER, NULL_STORE_START_ADDRESS); 212 | } 213 | 214 | ByteArrayInputStream inputstream = null; 215 | try { 216 | inputstream = new ByteArrayInputStream(bytes); 217 | boolean removed = IOHelper.readBoolean(inputstream); 218 | long size = IOHelper.read64bit(inputstream); 219 | int fileNumber = IOHelper.read32bit(inputstream); 220 | long startAddress = IOHelper.read64bit(inputstream); 221 | if (size == NULL_STORE_SIZE) { 222 | return null; 223 | } else { 224 | return new ContentStoreSlaveIndex(removed, size, slaveNumber, fileNumber, startAddress); 225 | } 226 | } catch (Throwable ex) { 227 | LOGGER.error("Read ContentStoreSlaveIndex failed. slaveNumber : " + slaveNumber, ex); 228 | return new ContentStoreSlaveIndex(true, TOTALLY_REMOVED_STORE_SIZE, slaveNumber, NULL_FILE_NUMBER, NULL_STORE_START_ADDRESS); 229 | } finally { 230 | Utils.closeInputStream(inputstream); 231 | } 232 | } else { 233 | if (bytes == null || bytes.length != STORE_SLAVE_INDEX_BYTES_LENGTH) { 234 | throw new IllegalArgumentException("Bytes length error. Expect length : " + STORE_SLAVE_INDEX_BYTES_LENGTH); 235 | } 236 | 237 | ByteArrayInputStream inputstream = null; 238 | try { 239 | inputstream = new ByteArrayInputStream(bytes); 240 | boolean removed = IOHelper.readBoolean(inputstream); 241 | long size = IOHelper.read64bit(inputstream); 242 | int fileNumber = IOHelper.read32bit(inputstream); 243 | long startAddress = IOHelper.read64bit(inputstream); 244 | if (size == NULL_STORE_SIZE) { 245 | return null; 246 | } else { 247 | return new ContentStoreSlaveIndex(removed, size, slaveNumber, fileNumber, startAddress); 248 | } 249 | } finally { 250 | Utils.closeInputStream(inputstream); 251 | } 252 | } 253 | } 254 | 255 | public static byte[] toBytes(ContentStoreSlaveIndex slaveIndex) throws IOException { 256 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 257 | if (slaveIndex == null) { 258 | IOHelper.writeBoolean(outputstream, false); 259 | IOHelper.write64bit(outputstream, NULL_STORE_SIZE); 260 | IOHelper.write32bit(outputstream, NULL_FILE_NUMBER); 261 | IOHelper.write64bit(outputstream, NULL_STORE_START_ADDRESS); 262 | } else { 263 | IOHelper.writeBoolean(outputstream, slaveIndex.isRemoved()); 264 | IOHelper.write64bit(outputstream, slaveIndex.getStoreSize()); 265 | IOHelper.write32bit(outputstream, slaveIndex.getFileNumber()); 266 | IOHelper.write64bit(outputstream, slaveIndex.getStoreStartAddress()); 267 | } 268 | 269 | outputstream.flush(); 270 | return outputstream.toByteArray(); 271 | } 272 | } 273 | 274 | } 275 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ContentStoreSlaveStoreFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.File; 20 | import java.io.RandomAccessFile; 21 | import java.nio.channels.FileChannel; 22 | import java.util.Collections; 23 | import java.util.LinkedList; 24 | import java.util.List; 25 | import java.util.concurrent.atomic.AtomicInteger; 26 | 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | /** 31 | * 32 | * @author Gary CHEN 33 | * 34 | */ 35 | class ContentStoreSlaveStoreFile { 36 | 37 | private final static Logger LOGGER = LoggerFactory.getLogger(ContentStoreSlaveStoreFile.class); 38 | 39 | //cache line padding 1 40 | private long p1, p2, p3, p4, p5, p6, p7, p8; 41 | 42 | private final ManagedStoreFileMetaData managedFileMeta; 43 | private volatile File slaveStoreFilePath; 44 | private volatile RandomAccessFile slaveStoreFile; 45 | private volatile FileChannel slaveStoreFileChannel; 46 | private final List slavesInThisFile; 47 | private int slavesInThisFileNumber; 48 | 49 | //cache line padding 2 50 | private long p9, p10, p11, p12, p13, p14, p15, p16; 51 | 52 | ContentStoreSlaveStoreFile(ManagedStoreFileMetaData managedFileMeta) { 53 | this.managedFileMeta = managedFileMeta; 54 | this.slavesInThisFile = new LinkedList(); 55 | this.slavesInThisFileNumber = 0; 56 | } 57 | 58 | ManagedStoreFileMetaData getManagedFileMeta() { 59 | return managedFileMeta; 60 | } 61 | 62 | File getSlaveStoreFilePath() { 63 | return slaveStoreFilePath; 64 | } 65 | 66 | void setSlaveStoreFilePath(File slaveStoreFilePath) { 67 | this.slaveStoreFilePath = slaveStoreFilePath; 68 | } 69 | 70 | RandomAccessFile getSlaveStoreFile() { 71 | return slaveStoreFile; 72 | } 73 | 74 | void setSlaveStoreFile(RandomAccessFile slaveStoreFile) { 75 | this.slaveStoreFile = slaveStoreFile; 76 | this.slaveStoreFileChannel = slaveStoreFile.getChannel(); 77 | } 78 | 79 | FileChannel getSlaveStoreFileChannel() { 80 | return slaveStoreFileChannel; 81 | } 82 | 83 | List getSlavesInThisFile() { 84 | return Collections.unmodifiableList(slavesInThisFile); 85 | } 86 | 87 | boolean addSlaveIndexInThisFile(ContentStoreSlaveIndex e) { 88 | return slavesInThisFile.add(e); 89 | } 90 | 91 | void clearSlavesInThisFile() { 92 | slavesInThisFile.clear(); 93 | } 94 | 95 | synchronized int getSlavesInThisFileNumber() { 96 | return slavesInThisFileNumber; 97 | } 98 | 99 | synchronized int incrementAndGetSlavesInThisFileNumber() { 100 | ++slavesInThisFileNumber; 101 | return slavesInThisFileNumber; 102 | } 103 | 104 | synchronized void resetSlavesInThisFileNumber() { 105 | slavesInThisFileNumber = 0; 106 | } 107 | 108 | synchronized int decrementAndGetSlavesInThisFileNumber() { 109 | --slavesInThisFileNumber; 110 | return slavesInThisFileNumber; 111 | } 112 | 113 | public long getP1() { 114 | return p1; 115 | } 116 | 117 | public long getP2() { 118 | return p2; 119 | } 120 | 121 | public long getP3() { 122 | return p3; 123 | } 124 | 125 | public long getP4() { 126 | return p4; 127 | } 128 | 129 | public long getP5() { 130 | return p5; 131 | } 132 | 133 | public long getP6() { 134 | return p6; 135 | } 136 | 137 | public long getP7() { 138 | return p7; 139 | } 140 | 141 | public long getP8() { 142 | return p8; 143 | } 144 | 145 | public long getP9() { 146 | return p9; 147 | } 148 | 149 | public long getP10() { 150 | return p10; 151 | } 152 | 153 | public long getP11() { 154 | return p11; 155 | } 156 | 157 | public long getP12() { 158 | return p12; 159 | } 160 | 161 | public long getP13() { 162 | return p13; 163 | } 164 | 165 | public long getP14() { 166 | return p14; 167 | } 168 | 169 | public long getP15() { 170 | return p15; 171 | } 172 | 173 | public long getP16() { 174 | return p16; 175 | } 176 | 177 | void closeFile() { 178 | try { 179 | slaveStoreFileChannel.force(true); 180 | } catch (Throwable ex) { 181 | if (LOGGER.isDebugEnabled()) { 182 | LOGGER.debug("Force write content to persistence file failed. File path: " + slaveStoreFilePath.getAbsolutePath() + ".", ex); 183 | } 184 | } 185 | 186 | try { 187 | slaveStoreFileChannel.close(); 188 | } catch (Throwable ex) { 189 | if (LOGGER.isDebugEnabled()) { 190 | LOGGER.debug("Close persistence file channel failed. File path: " + slaveStoreFilePath.getAbsolutePath() + ".", ex); 191 | } 192 | } 193 | 194 | try { 195 | slaveStoreFile.close(); 196 | } catch (Throwable ex) { 197 | if (LOGGER.isDebugEnabled()) { 198 | LOGGER.debug("Close persistence file failed. File path: " + slaveStoreFilePath.getAbsolutePath() + ".", ex); 199 | } 200 | } 201 | } 202 | } 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ErrorEntryRemovedListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | interface ErrorEntryRemovedListener { 25 | 26 | void entryRemoved(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ExceededMaxAllowedContentStoreSlavesNumber.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class ExceededMaxAllowedContentStoreSlavesNumber extends MallocContentStoreSpaceException { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 4204710639924251449L; 30 | 31 | /** 32 | * 33 | */ 34 | public ExceededMaxAllowedContentStoreSlavesNumber() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public ExceededMaxAllowedContentStoreSlavesNumber(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public ExceededMaxAllowedContentStoreSlavesNumber(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public ExceededMaxAllowedContentStoreSlavesNumber(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/FileBaseBinaryFileStoreConvertorImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.IOException; 20 | 21 | /** 22 | * 23 | * @author Gary CHEN 24 | * 25 | */ 26 | public class FileBaseBinaryFileStoreConvertorImpl implements BinaryFileStoreConvertor { 27 | 28 | public FileBaseBinaryFileStoreConvertorImpl() { 29 | } 30 | 31 | /* (non-Javadoc) 32 | */ 33 | @Override 34 | public BinaryFileStoreValue fromStoreBlock(ContentStoreBlock block) throws ContentStoreBlockException { 35 | if (!(block instanceof FileBaseContentStoreBlockImpl)) { 36 | throw new IllegalArgumentException("ContentStoreBlock type error. Expected Type: " + FileBaseContentStoreBlockImpl.class.getName()); 37 | } 38 | 39 | try { 40 | FileBaseContentStoreBlockImpl blockImpl = (FileBaseContentStoreBlockImpl) block; 41 | byte[] valueBytes = blockImpl.getPayload(); 42 | FileBaseBinaryFileStoreValueImpl valueImpl = FileBaseBinaryFileStoreValueImpl.fromBytes(valueBytes); 43 | return valueImpl; 44 | } catch (Throwable ex) { 45 | throw new ContentStoreBlockException("Convert bytes to FileBaseBinaryFileStoreValueImpl failed.", ex); 46 | } 47 | } 48 | 49 | /* (non-Javadoc) 50 | */ 51 | @Override 52 | public ContentStoreBlock toStoreBlock(BinaryFileStoreValue storeValue) throws ContentStoreBlockException { 53 | if (!(storeValue instanceof FileBaseBinaryFileStoreValueImpl)) { 54 | throw new IllegalArgumentException("BinaryFileStoreValue type error. Expected Type: " + FileBaseBinaryFileStoreValueImpl.class.getName()); 55 | } 56 | 57 | try { 58 | FileBaseBinaryFileStoreValueImpl valueImpl = (FileBaseBinaryFileStoreValueImpl) storeValue; 59 | byte[] valueBytes = FileBaseBinaryFileStoreValueImpl.toBytes(valueImpl); 60 | FileBaseContentStoreBlockImpl blockImpl = new FileBaseContentStoreBlockImpl(); 61 | blockImpl.setPayload(valueBytes); 62 | return blockImpl; 63 | } catch (IOException ex) { 64 | throw new ContentStoreBlockException("Convert FileBaseBinaryFileStoreValueImpl to bytes failed.", ex); 65 | } 66 | } 67 | 68 | } 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/FileBaseBinaryFileStoreKeyImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.Externalizable; 22 | import java.io.IOException; 23 | 24 | import com.github.garychenc.filemap.util.*; 25 | 26 | /** 27 | * 28 | * @author Gary CHEN 29 | * 30 | */ 31 | public class FileBaseBinaryFileStoreKeyImpl implements BinaryFileStoreKey { 32 | 33 | private final static int CLASS_VERSION = 1; 34 | private final static String NULL_OBJECT = "null"; 35 | private final static String UseExternalizable = "UseExternalizable"; 36 | private final static String UseSerializable = "UseSerializable"; 37 | 38 | private volatile int hash = -1; 39 | private final StoreKey realKey; 40 | 41 | public FileBaseBinaryFileStoreKeyImpl(StoreKey realKey) { 42 | Asserts.notNull("realKey", realKey); 43 | this.realKey = realKey; 44 | } 45 | 46 | /* (non-Javadoc) 47 | */ 48 | @Override 49 | public int getHashValue() { 50 | return hash; 51 | } 52 | 53 | /* (non-Javadoc) 54 | */ 55 | @Override 56 | public void setHashValue(int hash) { 57 | this.hash = hash; 58 | } 59 | 60 | /* (non-Javadoc) 61 | */ 62 | @Override 63 | public String getUuid() { 64 | return realKey.getUuid(); 65 | } 66 | 67 | @Override 68 | public int hashCode() { 69 | final int prime = 31; 70 | int result = 1; 71 | result = prime * result + ((realKey == null) ? 0 : realKey.hashCode()); 72 | return result; 73 | } 74 | 75 | @Override 76 | public boolean equals(Object obj) { 77 | if (this == obj) 78 | return true; 79 | if (obj == null) 80 | return false; 81 | if (getClass() != obj.getClass()) 82 | return false; 83 | FileBaseBinaryFileStoreKeyImpl other = (FileBaseBinaryFileStoreKeyImpl) obj; 84 | if (realKey == null) { 85 | if (other.realKey != null) 86 | return false; 87 | } else if (!realKey.equals(other.realKey)) 88 | return false; 89 | return true; 90 | } 91 | 92 | @Override 93 | public String toString() { 94 | return "FileBaseBinaryFileStoreKeyImpl [realKey=" + realKey.toString() + "]"; 95 | } 96 | 97 | private final static byte[] ZERO_BYTES = new byte[0]; 98 | 99 | public static final byte[] toBytes(FileBaseBinaryFileStoreKeyImpl key) throws IOException { 100 | if (key == null) { 101 | return ZERO_BYTES; 102 | } 103 | 104 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 105 | IOHelper.writeClassVersion(outputstream, CLASS_VERSION); 106 | IOHelper.write32bit(outputstream, key.hash); 107 | 108 | if (key.realKey == null) { 109 | IOHelper.writeUTF32(outputstream, NULL_OBJECT); 110 | } else { 111 | if (key.realKey instanceof Externalizable) { 112 | IOHelper.writeUTF32(outputstream, UseExternalizable); 113 | Externalizable v = (Externalizable) key.realKey; 114 | 115 | try (ByteArrayOutputStream outputstreamForExternalizable = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 116 | IOHelper.writeUTF(outputstreamForExternalizable, v.getClass().getName()); 117 | ByteArrayObjectOutput out = new ByteArrayObjectOutput(outputstreamForExternalizable, true); 118 | v.writeExternal(out); 119 | outputstreamForExternalizable.flush(); 120 | IOHelper.writeByteArray32(outputstream, outputstreamForExternalizable.toByteArray()); 121 | } 122 | } else { 123 | IOHelper.writeUTF32(outputstream, UseSerializable); 124 | 125 | try (ByteArrayOutputStream outputstreamForSerializable = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 126 | IOHelper.writeObject(outputstreamForSerializable, key.realKey); 127 | outputstreamForSerializable.flush(); 128 | IOHelper.writeByteArray32(outputstream, outputstreamForSerializable.toByteArray()); 129 | } 130 | } 131 | } 132 | 133 | outputstream.flush(); 134 | return outputstream.toByteArray(); 135 | } 136 | } 137 | 138 | public static final FileBaseBinaryFileStoreKeyImpl fromBytes(byte[] bytes) throws Exception { 139 | if (bytes == null || bytes.length == 0) { 140 | return null; 141 | } 142 | 143 | ByteArrayInputStream inputstream = new ByteArrayInputStream(bytes); 144 | IOHelper.readClassVersion(inputstream); 145 | int hashValue = IOHelper.read32bit(inputstream); 146 | String realKeyFlag = IOHelper.readUTF32(inputstream); 147 | StoreKey key = null; 148 | if (NULL_OBJECT.equals(realKeyFlag)) { 149 | key = null; 150 | } else if (UseExternalizable.equals(realKeyFlag)) { 151 | byte[] realKeyBytes = IOHelper.readByteArray32(inputstream); 152 | ByteArrayInputStream externalizableInputstream = new ByteArrayInputStream(realKeyBytes); 153 | String className = IOHelper.readUTF(externalizableInputstream); 154 | Object obj = Utils.createObject(className); 155 | ByteArrayObjectInput in = new ByteArrayObjectInput(externalizableInputstream, true); 156 | ((Externalizable) obj).readExternal(in); 157 | key = (StoreKey)obj; 158 | } else if (UseSerializable.equals(realKeyFlag)) { 159 | byte[] realKeyBytes = IOHelper.readByteArray32(inputstream); 160 | ByteArrayInputStream serializableInputstream = new ByteArrayInputStream(realKeyBytes); 161 | key = (StoreKey)IOHelper.readObject(serializableInputstream); 162 | } else { 163 | throw new IOException("Real key flag broken. Flag value: " + realKeyFlag); 164 | } 165 | 166 | FileBaseBinaryFileStoreKeyImpl storeKeyImpl = new FileBaseBinaryFileStoreKeyImpl(key); 167 | storeKeyImpl.setHashValue(hashValue); 168 | return storeKeyImpl; 169 | } 170 | 171 | public StoreKey getRealKey() { 172 | return realKey; 173 | } 174 | 175 | @Override 176 | public long getVersionNumber() { 177 | return 0; 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/FileBaseBinaryFileStoreValueImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.Externalizable; 22 | import java.io.IOException; 23 | 24 | import com.github.garychenc.filemap.util.*; 25 | 26 | /** 27 | * 28 | * @author Gary CHEN 29 | * 30 | */ 31 | public class FileBaseBinaryFileStoreValueImpl implements BinaryFileStoreValue { 32 | 33 | private final static int CLASS_VERSION = 1; 34 | private final static String NULL_OBJECT = "null"; 35 | private final static String NOT_NULL_OBJECT = "not_null"; 36 | private final static String UseExternalizable = "UseExternalizable"; 37 | private final static String UseSerializable = "UseSerializable"; 38 | private final static String UseByteArray = "UseByteArray"; 39 | private final static String UseString = "UseString"; 40 | 41 | private volatile FileBaseBinaryFileStoreKeyImpl originalStoreKey = null; 42 | private volatile long versionNumber = 0; 43 | private Object realValue; 44 | 45 | public FileBaseBinaryFileStoreValueImpl(Object realValue) { 46 | this.realValue = realValue; 47 | } 48 | 49 | /* (non-Javadoc) 50 | */ 51 | @Override 52 | public StoreKey getOriginalStoreKey() { 53 | return originalStoreKey; 54 | } 55 | 56 | @Override 57 | public long getVersionNumber() { 58 | return versionNumber; 59 | } 60 | 61 | @Override 62 | public void setVersionNumber(long versionNumber) { 63 | this.versionNumber = versionNumber; 64 | } 65 | 66 | /* (non-Javadoc) 67 | */ 68 | @Override 69 | public void setOriginalStoreKey(StoreKey key) { 70 | if (key == null) { 71 | this.originalStoreKey = null; 72 | } else { 73 | if (!(key instanceof FileBaseBinaryFileStoreKeyImpl)) { 74 | throw new IllegalArgumentException("StoreKey must be the type of " + FileBaseBinaryFileStoreKeyImpl.class.getName()); 75 | } 76 | 77 | this.originalStoreKey = (FileBaseBinaryFileStoreKeyImpl)key; 78 | } 79 | } 80 | 81 | private final static byte[] ZERO_BYTES = new byte[0]; 82 | 83 | public static final byte[] toBytes(FileBaseBinaryFileStoreValueImpl value) throws IOException { 84 | if (value == null) { 85 | return ZERO_BYTES; 86 | } 87 | 88 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 89 | IOHelper.writeClassVersion(outputstream, CLASS_VERSION); 90 | 91 | IOHelper.write64bit(outputstream, value.versionNumber); 92 | 93 | if (value.originalStoreKey == null) { 94 | IOHelper.writeUTF32(outputstream, NULL_OBJECT); 95 | } else { 96 | IOHelper.writeUTF32(outputstream, NOT_NULL_OBJECT); 97 | byte[] keyBytes = FileBaseBinaryFileStoreKeyImpl.toBytes(value.originalStoreKey); 98 | if (keyBytes == null) { 99 | throw new IllegalStateException("Bytes of originalStoreKey can not be null."); 100 | } 101 | 102 | IOHelper.writeByteArray32(outputstream, keyBytes); 103 | } 104 | 105 | if (value.realValue == null) { 106 | IOHelper.writeUTF32(outputstream, NULL_OBJECT); 107 | } else { 108 | if (value.realValue instanceof byte[]) { 109 | IOHelper.writeUTF32(outputstream, UseByteArray); 110 | byte[] v = (byte[]) value.realValue; 111 | IOHelper.writeByteArray32(outputstream, v); 112 | } else if (value.realValue instanceof String) { 113 | IOHelper.writeUTF32(outputstream, UseString); 114 | IOHelper.writeUTF32(outputstream, (String) value.realValue); 115 | } else if (value.realValue instanceof Externalizable) { 116 | IOHelper.writeUTF32(outputstream, UseExternalizable); 117 | Externalizable v = (Externalizable) value.realValue; 118 | 119 | try (ByteArrayOutputStream outputstreamForExternalizable = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 120 | IOHelper.writeUTF(outputstreamForExternalizable, v.getClass().getName()); 121 | ByteArrayObjectOutput out = new ByteArrayObjectOutput(outputstreamForExternalizable, true); 122 | v.writeExternal(out); 123 | outputstreamForExternalizable.flush(); 124 | IOHelper.writeByteArray32(outputstream, outputstreamForExternalizable.toByteArray()); 125 | } 126 | } else { 127 | IOHelper.writeUTF32(outputstream, UseSerializable); 128 | 129 | try (ByteArrayOutputStream outputstreamForSerializable = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 130 | IOHelper.writeObject(outputstreamForSerializable, value.realValue); 131 | outputstreamForSerializable.flush(); 132 | IOHelper.writeByteArray32(outputstream, outputstreamForSerializable.toByteArray()); 133 | } 134 | } 135 | } 136 | 137 | outputstream.flush(); 138 | return outputstream.toByteArray(); 139 | } 140 | } 141 | 142 | public static final FileBaseBinaryFileStoreValueImpl fromBytes(byte[] bytes) throws Exception { 143 | if (bytes == null || bytes.length == 0) { 144 | return null; 145 | } 146 | 147 | ByteArrayInputStream inputstream = new ByteArrayInputStream(bytes); 148 | IOHelper.readClassVersion(inputstream); 149 | 150 | long versionNumber = IOHelper.read64bit(inputstream); 151 | 152 | String storeKeyFlag = IOHelper.readUTF32(inputstream); 153 | FileBaseBinaryFileStoreKeyImpl storeKey = null; 154 | if (NULL_OBJECT.equals(storeKeyFlag)) { 155 | storeKey = null; 156 | } else { 157 | byte[] keyBytes = IOHelper.readByteArray32(inputstream); 158 | storeKey = FileBaseBinaryFileStoreKeyImpl.fromBytes(keyBytes); 159 | } 160 | 161 | String realValueFlag = IOHelper.readUTF32(inputstream); 162 | Object v = null; 163 | if (NULL_OBJECT.equals(realValueFlag)) { 164 | v = null; 165 | } else if (UseByteArray.equals(realValueFlag)) { 166 | v = IOHelper.readByteArray32(inputstream); 167 | } else if (UseString.equals(realValueFlag)) { 168 | v = IOHelper.readUTF32(inputstream); 169 | } else if (UseExternalizable.equals(realValueFlag)) { 170 | byte[] valueBytes = IOHelper.readByteArray32(inputstream); 171 | ByteArrayInputStream externalizableInputstream = new ByteArrayInputStream(valueBytes); 172 | String className = IOHelper.readUTF(externalizableInputstream); 173 | Object obj = Utils.createObject(className); 174 | ByteArrayObjectInput in = new ByteArrayObjectInput(externalizableInputstream, true); 175 | ((Externalizable) obj).readExternal(in); 176 | v = obj; 177 | } else if (UseSerializable.equals(realValueFlag)) { 178 | byte[] valueBytes = IOHelper.readByteArray32(inputstream); 179 | ByteArrayInputStream serializableInputstream = new ByteArrayInputStream(valueBytes); 180 | v = IOHelper.readObject(serializableInputstream); 181 | } else { 182 | throw new IOException("Real value flag broken. Flag value: " + realValueFlag); 183 | } 184 | 185 | FileBaseBinaryFileStoreValueImpl storeValue = new FileBaseBinaryFileStoreValueImpl(v); 186 | storeValue.setOriginalStoreKey(storeKey); 187 | storeValue.setVersionNumber(versionNumber); 188 | return storeValue; 189 | } 190 | 191 | /** 192 | * @return the realValue 193 | */ 194 | public Object getRealValue() { 195 | return realValue; 196 | } 197 | 198 | public void setRealValue(Object realValue) { 199 | this.realValue = realValue; 200 | } 201 | } 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/FileBaseContentStoreBlockImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | 23 | import com.github.garychenc.filemap.util.IOHelper; 24 | 25 | /** 26 | * 27 | * @author Gary CHEN 28 | * 29 | */ 30 | public class FileBaseContentStoreBlockImpl implements ContentStoreBlock { 31 | 32 | private static final int CLASS_VERSION = 1; 33 | private static final String NULL_OBJECT = "null"; 34 | private static final String NOT_NULL_OBJECT = "not_null"; 35 | 36 | public static final int BLOCK_LENGTH_OFFSET = 2; 37 | public static final int BLOCK_LENGTH_STORE_LENGTH = 8; 38 | 39 | public static final int REMOVED_OFFSET = 10; 40 | public static final int REMOVED_STORE_LENGTH = 1; 41 | public static final byte UNREMOVED_MARK = 0x00; 42 | public static final byte REMOVED_MARK = 0x01; 43 | 44 | public static final long BLOCK_LEAST_LENGTH = 11L; 45 | 46 | private volatile byte[] payload = null; // need to be persisted 47 | private volatile long blockLength = -1; // need to be persisted 48 | private volatile boolean removed = false; // need to be persisted 49 | 50 | public FileBaseContentStoreBlockImpl() { 51 | } 52 | 53 | /* (non-Javadoc) 54 | */ 55 | @Override 56 | public long getContentLength() throws ContentStoreBlockException { 57 | if (blockLength == -1) { 58 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 59 | IOHelper.writeClassVersion(outputstream, CLASS_VERSION); 60 | IOHelper.write64bit(outputstream, blockLength); 61 | IOHelper.writeBoolean(outputstream, removed); 62 | writePayload(outputstream); 63 | outputstream.flush(); 64 | blockLength = outputstream.size(); 65 | } catch (IOException ex) { 66 | throw new ContentStoreBlockException("Write block content to bytes failed.", ex); 67 | } 68 | } 69 | 70 | return blockLength; 71 | } 72 | 73 | /* (non-Javadoc) 74 | */ 75 | @Override 76 | public byte[] toBytes() throws ContentStoreBlockException { 77 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 78 | IOHelper.writeClassVersion(outputstream, CLASS_VERSION); 79 | IOHelper.write64bit(outputstream, this.getContentLength()); 80 | IOHelper.writeBoolean(outputstream, removed); 81 | writePayload(outputstream); 82 | outputstream.flush(); 83 | return outputstream.toByteArray(); 84 | } catch (IOException ex) { 85 | throw new ContentStoreBlockException("Write block content to bytes failed.", ex); 86 | } 87 | } 88 | 89 | @Override 90 | public void fromBytes(byte[] bytes) throws ContentStoreBlockException { 91 | if (bytes == null || bytes.length == 0) { 92 | return; 93 | } 94 | 95 | try { 96 | ByteArrayInputStream inputstream = new ByteArrayInputStream(bytes); 97 | IOHelper.readClassVersion(inputstream); 98 | blockLength = IOHelper.read64bit(inputstream); 99 | removed = IOHelper.readBoolean(inputstream); 100 | readPayload(inputstream); 101 | } catch (IOException ex) { 102 | throw new ContentStoreBlockException("Read block content from bytes failed.", ex); 103 | } 104 | } 105 | 106 | private void writePayload(ByteArrayOutputStream outputstream) throws IOException { 107 | if (payload == null) { 108 | IOHelper.writeUTF32(outputstream, NULL_OBJECT); 109 | } else { 110 | IOHelper.writeUTF32(outputstream, NOT_NULL_OBJECT); 111 | IOHelper.writeByteArray32(outputstream, payload); 112 | } 113 | } 114 | 115 | private void readPayload(ByteArrayInputStream inputstream) throws IOException { 116 | String flag = IOHelper.readUTF32(inputstream); 117 | if (NULL_OBJECT.equals(flag)) { 118 | payload = null; 119 | } else { 120 | payload = IOHelper.readByteArray32(inputstream); 121 | } 122 | } 123 | 124 | public byte[] getPayload() { 125 | return payload; 126 | } 127 | 128 | public byte[] setPayload(byte[] payload) { 129 | resetBlockLength(); 130 | byte[] oldPayload = this.payload; 131 | this.payload = payload; 132 | return oldPayload; 133 | } 134 | 135 | public FileBaseContentStoreBlockImpl copy() { 136 | FileBaseContentStoreBlockImpl block = new FileBaseContentStoreBlockImpl(); 137 | block.setPayload(payload); 138 | return block; 139 | } 140 | 141 | private void resetBlockLength() { 142 | blockLength = -1; 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/FileBaseContentStoreSlave.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import static com.github.garychenc.filemap.util.Asserts.*; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.IOException; 23 | import java.nio.ByteBuffer; 24 | import java.util.concurrent.atomic.AtomicBoolean; 25 | import java.util.concurrent.locks.ReadWriteLock; 26 | import java.util.concurrent.locks.ReentrantReadWriteLock; 27 | 28 | import com.github.garychenc.filemap.util.IOHelper; 29 | import com.github.garychenc.filemap.util.Utils; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * 35 | * @author Gary CHEN 36 | * 37 | */ 38 | public class FileBaseContentStoreSlave implements ContentStoreSlave { 39 | 40 | //TODO : add a cache in this level to improve the performance of searching 41 | private final static Logger LOGGER = LoggerFactory.getLogger(FileBaseContentStoreSlave.class); 42 | 43 | //cache line padding 1 44 | private long p1, p2, p3, p4, p5, p6, p7, p8; 45 | 46 | private final FileBaseContentStoreMaster master; 47 | private final ContentStoreSlaveStoreFile file; 48 | private final ContentStoreSlaveIndex slaveIndex; 49 | 50 | private volatile boolean closed; 51 | private final ReadWriteLock slaveLock; 52 | 53 | //cache line padding 2 54 | private long p9, p10, p11, p12, p13, p14, p15, p16; 55 | 56 | public FileBaseContentStoreSlave(ContentStoreSlaveStoreFile file, FileBaseContentStoreMaster master, ContentStoreSlaveIndex slaveIndex) { 57 | notNull("master", master); 58 | notNull("slaveIndex", slaveIndex); 59 | notNull("file", file); 60 | 61 | this.master = master; 62 | this.slaveIndex = slaveIndex; 63 | 64 | this.file = file; 65 | this.closed = false; 66 | this.slaveLock = new ReentrantReadWriteLock(); 67 | } 68 | 69 | private void checkClosed() throws ContentStoreSlaveException { 70 | if (closed) { 71 | throw new ContentStoreSlaveException("ContentStoreSlave had been closed. slaveNumber = " + slaveIndex.getSlaveNumber()); 72 | } 73 | } 74 | 75 | /* (non-Javadoc) 76 | */ 77 | @Override 78 | public void delete() throws ContentStoreSlaveException { 79 | slaveLock.writeLock().lock(); 80 | try { 81 | checkClosed(); 82 | 83 | byte[] bytes = new byte[FileBaseContentStoreBlockImpl.REMOVED_STORE_LENGTH]; 84 | bytes[0] = FileBaseContentStoreBlockImpl.REMOVED_MARK; 85 | file.getSlaveStoreFileChannel().write(ByteBuffer.wrap(bytes), slaveIndex.getStoreStartAddress() + FileBaseContentStoreBlockImpl.REMOVED_OFFSET); 86 | if (master.isForceWriteData()) { 87 | file.getSlaveStoreFileChannel().force(true); 88 | } 89 | 90 | master.removeSlave(slaveIndex.getSlaveNumber()); 91 | } catch (IOException ex) { 92 | throw new ContentStoreSlaveException("Mark FileBaseContentStoreBlockImpl as deleted failed. Slave store file : " + file.getSlaveStoreFilePath().getAbsolutePath() + ", Start Address: " + slaveIndex.getStoreStartAddress(), ex); 93 | } catch (ContentStoreMasterException ex) { 94 | try { 95 | byte[] bytes = new byte[FileBaseContentStoreBlockImpl.REMOVED_STORE_LENGTH]; 96 | bytes[0] = FileBaseContentStoreBlockImpl.UNREMOVED_MARK; 97 | file.getSlaveStoreFileChannel().write(ByteBuffer.wrap(bytes), slaveIndex.getStoreStartAddress() + FileBaseContentStoreBlockImpl.REMOVED_OFFSET); 98 | if (master.isForceWriteData()) { 99 | file.getSlaveStoreFileChannel().force(true); 100 | } 101 | } catch (IOException exce) { 102 | LOGGER.error("Undo mark FileBaseContentStoreBlockImpl as deleted failed.", exce); 103 | } 104 | 105 | throw new ContentStoreSlaveException("Remove slave from master failed. Slave Number : " + slaveIndex.getSlaveNumber(), ex); 106 | } finally { 107 | slaveLock.writeLock().unlock(); 108 | } 109 | } 110 | 111 | /* (non-Javadoc) 112 | */ 113 | @Override 114 | public long getSlaveNumber() { 115 | return slaveIndex.getSlaveNumber(); 116 | } 117 | 118 | /* (non-Javadoc) 119 | */ 120 | @Override 121 | public ContentStoreBlock read() throws ContentStoreSlaveException { 122 | slaveLock.readLock().lock(); 123 | try { 124 | if (closed) { 125 | throw new IllegalStateException("Store slave had been closed."); 126 | } 127 | 128 | byte[] bytes = new byte[(int)slaveIndex.getStoreSize()]; 129 | IOHelper.readFully(file.getSlaveStoreFileChannel(), bytes, slaveIndex.getStoreStartAddress()); 130 | boolean removed = bytes[FileBaseContentStoreBlockImpl.REMOVED_OFFSET] != FileBaseContentStoreBlockImpl.UNREMOVED_MARK; 131 | if (removed) { 132 | return null; 133 | } 134 | 135 | ByteArrayInputStream inputstream4BlockLength = new ByteArrayInputStream(bytes, FileBaseContentStoreBlockImpl.BLOCK_LENGTH_OFFSET, FileBaseContentStoreBlockImpl.BLOCK_LENGTH_STORE_LENGTH); 136 | long blockLength = IOHelper.read64bit(inputstream4BlockLength); 137 | if (blockLength < FileBaseContentStoreBlockImpl.BLOCK_LEAST_LENGTH) { 138 | throw new ContentStoreSlaveException("ContentStoreBlock length error. min length : " + FileBaseContentStoreBlockImpl.BLOCK_LEAST_LENGTH + ", current length: " + blockLength); 139 | } 140 | 141 | if (blockLength > bytes.length) { 142 | throw new ContentStoreSlaveException("ContentStoreBlock length error. Max length : " + bytes.length + ", current length: " + blockLength); 143 | } 144 | 145 | FileBaseContentStoreBlockImpl fileBlock = new FileBaseContentStoreBlockImpl(); 146 | fileBlock.fromBytes(Utils.copyOfRange(bytes, 0, (int)blockLength)); 147 | return fileBlock; 148 | } catch (IOException ex) { 149 | throw new ContentStoreSlaveException("Read FileBaseContentStoreBlockImpl failed. Slave store file : " + file.getSlaveStoreFilePath().getAbsolutePath() + ", Start Address: " + slaveIndex.getStoreStartAddress(), ex); 150 | } catch (ContentStoreBlockException ex) { 151 | throw new ContentStoreSlaveException("Convert bytes to FileBaseContentStoreBlockImpl failed.", ex); 152 | } finally { 153 | slaveLock.readLock().unlock(); 154 | } 155 | } 156 | 157 | /* (non-Javadoc) 158 | */ 159 | @Override 160 | public void store(ContentStoreBlock block) throws ContentStoreSlaveException, ContentStoreBlockLengthExceedException { 161 | notNull("block", block); 162 | if (!(block instanceof FileBaseContentStoreBlockImpl)) { 163 | throw new ContentStoreSlaveException("ContentStoreBlock type error. Expect Type : " + FileBaseContentStoreBlockImpl.class.getName()); 164 | } 165 | 166 | slaveLock.writeLock().lock(); 167 | try { 168 | checkClosed(); 169 | 170 | byte[] blockBytes = block.toBytes(); 171 | if (blockBytes.length != block.getContentLength()) { 172 | throw new ContentStoreSlaveException("ContentStoreBlock length error. block.toBytes().length = " + blockBytes.length + ", block.getContentLength() = " + block.getContentLength() + "."); 173 | } 174 | 175 | long blockLength = block.getContentLength(); 176 | if (blockLength > slaveIndex.getStoreSize()) { 177 | throw new ContentStoreBlockLengthExceedException("ContentStoreBlock Length: " + blockLength + ", Store Size: " + slaveIndex.getStoreSize() + "."); 178 | } 179 | 180 | file.getSlaveStoreFileChannel().write(ByteBuffer.wrap(blockBytes), slaveIndex.getStoreStartAddress()); 181 | if (master.isForceWriteData()) { 182 | file.getSlaveStoreFileChannel().force(true); 183 | } 184 | } catch (ContentStoreBlockException ex) { 185 | throw new ContentStoreSlaveException("Convert block to bytes failed.", ex); 186 | } catch (IOException ex) { 187 | throw new ContentStoreSlaveException("Store block bytes to slave store file failed. Slave store file : " + file.getSlaveStoreFilePath().getAbsolutePath() + ", Start Address: " + slaveIndex.getStoreStartAddress(), ex); 188 | } finally { 189 | slaveLock.writeLock().unlock(); 190 | } 191 | } 192 | 193 | public long getStoreSize() { 194 | return slaveIndex.getStoreSize(); 195 | } 196 | 197 | public void close() { 198 | slaveLock.writeLock().lock(); 199 | try { 200 | this.closed = true; 201 | } finally { 202 | slaveLock.writeLock().unlock(); 203 | } 204 | } 205 | 206 | public void reopen() { 207 | slaveLock.writeLock().lock(); 208 | try { 209 | this.closed = false; 210 | } finally { 211 | slaveLock.writeLock().unlock(); 212 | } 213 | } 214 | 215 | public boolean isClosed() { 216 | slaveLock.readLock().lock(); 217 | try { 218 | return closed; 219 | } finally { 220 | slaveLock.readLock().unlock(); 221 | } 222 | } 223 | 224 | public long getP1() { 225 | return p1; 226 | } 227 | 228 | public long getP2() { 229 | return p2; 230 | } 231 | 232 | public long getP3() { 233 | return p3; 234 | } 235 | 236 | public long getP4() { 237 | return p4; 238 | } 239 | 240 | public long getP5() { 241 | return p5; 242 | } 243 | 244 | public long getP6() { 245 | return p6; 246 | } 247 | 248 | public long getP7() { 249 | return p7; 250 | } 251 | 252 | public long getP8() { 253 | return p8; 254 | } 255 | 256 | public long getP9() { 257 | return p9; 258 | } 259 | 260 | public long getP10() { 261 | return p10; 262 | } 263 | 264 | public long getP11() { 265 | return p11; 266 | } 267 | 268 | public long getP12() { 269 | return p12; 270 | } 271 | 272 | public long getP13() { 273 | return p13; 274 | } 275 | 276 | public long getP14() { 277 | return p14; 278 | } 279 | 280 | public long getP15() { 281 | return p15; 282 | } 283 | 284 | public long getP16() { 285 | return p16; 286 | } 287 | 288 | public int getFileNumber() { 289 | return slaveIndex.getFileNumber(); 290 | } 291 | 292 | public long getStoreStartAddress() { 293 | return slaveIndex.getStoreStartAddress(); 294 | } 295 | 296 | public ContentStoreSlaveIndex getSlaveIndex() { 297 | return slaveIndex; 298 | } 299 | 300 | } 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/IndexBucket.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | 23 | import com.github.garychenc.filemap.util.IOHelper; 24 | import com.github.garychenc.filemap.util.Utils; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | /** 29 | * 30 | * @author Gary CHEN 31 | * 32 | */ 33 | class IndexBucket { 34 | 35 | private final static Logger LOGGER = LoggerFactory.getLogger(IndexBucket.class); 36 | 37 | final static int BUCKET_BYTES_LENGTH = 13; 38 | final static byte[] NULL_BUCKET_BYTES; 39 | 40 | final static long REMOVED_SLAVE_NUMBER = -13; 41 | final static long NULL_SLAVE_NUMBER = -1; 42 | final static int NULL_HASH_VALUE = -1; 43 | 44 | static { 45 | ByteArrayOutputStream outputstream = null; 46 | try { 47 | outputstream = new ByteArrayOutputStream(); 48 | IOHelper.writeBoolean(outputstream, false); 49 | IOHelper.write64bit(outputstream, NULL_SLAVE_NUMBER); 50 | IOHelper.write32bit(outputstream, NULL_HASH_VALUE); 51 | outputstream.flush(); 52 | NULL_BUCKET_BYTES = outputstream.toByteArray(); 53 | } catch (Throwable ex) { 54 | throw new IllegalStateException("Write null bucket bytes failed.", ex); 55 | } finally { 56 | if (outputstream != null) { 57 | try { outputstream.close(); } catch (IOException exce) { } 58 | } 59 | } 60 | } 61 | 62 | //cache line padding 1 63 | private long p1, p2, p3, p4, p5, p6, p7, p8; 64 | 65 | private volatile long slaveNumber; 66 | private final int bucketIndex; 67 | 68 | //cache line padding 2 69 | private long p9, p10, p11, p12, p13, p14, p15, p16; 70 | 71 | IndexBucket(long slaveNumber, int bucketIndex) { 72 | this.slaveNumber = slaveNumber; 73 | this.bucketIndex = bucketIndex; 74 | } 75 | 76 | void setSlaveNumber(long slaveNumber) { 77 | this.slaveNumber = slaveNumber; 78 | } 79 | 80 | long getSlaveNumber() { 81 | return slaveNumber; 82 | } 83 | 84 | int getBucketIndex() { 85 | return bucketIndex; 86 | } 87 | 88 | static final class BucketFromBytesResult { 89 | private final IndexBucket bucket; 90 | private final boolean removed; 91 | private final int hashValue; 92 | 93 | private BucketFromBytesResult(IndexBucket bucket, boolean removed, int hashValue) { 94 | this.bucket = bucket; 95 | this.removed = removed; 96 | this.hashValue = hashValue; 97 | } 98 | 99 | IndexBucket getBucket() { 100 | return bucket; 101 | } 102 | 103 | boolean isRemoved() { 104 | return removed; 105 | } 106 | 107 | int getHashValue() { 108 | return hashValue; 109 | } 110 | } 111 | 112 | static BucketFromBytesResult fromBytes(byte[] bytes, int bucketIndex, boolean ignoreErrorEntry) throws IOException { 113 | if (ignoreErrorEntry) { 114 | if (bytes == null || bytes.length != BUCKET_BYTES_LENGTH) { 115 | IndexBucket bucket = new IndexBucket(REMOVED_SLAVE_NUMBER, bucketIndex); 116 | return new BucketFromBytesResult(bucket, true, NULL_HASH_VALUE); 117 | } 118 | 119 | ByteArrayInputStream inputstream = null; 120 | try { 121 | inputstream = new ByteArrayInputStream(bytes); 122 | boolean removed = IOHelper.readBoolean(inputstream); 123 | long number = IOHelper.read64bit(inputstream); 124 | int hash = IOHelper.read32bit(inputstream); 125 | if (number == NULL_SLAVE_NUMBER) { 126 | return null; 127 | } else { 128 | IndexBucket bucket = new IndexBucket(number, bucketIndex); 129 | return new BucketFromBytesResult(bucket, removed, hash); 130 | } 131 | } catch (Throwable ex) { 132 | LOGGER.error("Read IndexBucket failed. bucketIndex : " + bucketIndex, ex); 133 | IndexBucket bucket = new IndexBucket(REMOVED_SLAVE_NUMBER, bucketIndex); 134 | return new BucketFromBytesResult(bucket, true, NULL_HASH_VALUE); 135 | } finally { 136 | Utils.closeInputStream(inputstream); 137 | } 138 | } else { 139 | if (bytes == null || bytes.length != BUCKET_BYTES_LENGTH) { 140 | throw new IllegalArgumentException("Bytes length error. Expect length : " + BUCKET_BYTES_LENGTH); 141 | } 142 | 143 | ByteArrayInputStream inputstream = null; 144 | try { 145 | inputstream = new ByteArrayInputStream(bytes); 146 | boolean removed = IOHelper.readBoolean(inputstream); 147 | long number = IOHelper.read64bit(inputstream); 148 | int hash = IOHelper.read32bit(inputstream); 149 | if (number == NULL_SLAVE_NUMBER) { 150 | return null; 151 | } else { 152 | IndexBucket bucket = new IndexBucket(number, bucketIndex); 153 | return new BucketFromBytesResult(bucket, removed, hash); 154 | } 155 | } finally { 156 | Utils.closeInputStream(inputstream); 157 | } 158 | } 159 | } 160 | 161 | static byte[] toBytes(IndexBucket bucket, boolean removed, int hashValue) throws IOException { 162 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 163 | if (bucket == null) { 164 | IOHelper.writeBoolean(outputstream, false); 165 | IOHelper.write64bit(outputstream, NULL_SLAVE_NUMBER); 166 | IOHelper.write32bit(outputstream, NULL_HASH_VALUE); 167 | } else { 168 | IOHelper.writeBoolean(outputstream, removed); 169 | IOHelper.write64bit(outputstream, bucket.slaveNumber); 170 | IOHelper.write32bit(outputstream, hashValue); 171 | } 172 | 173 | outputstream.flush(); 174 | return outputstream.toByteArray(); 175 | } 176 | } 177 | 178 | public long getP1() { 179 | return p1; 180 | } 181 | 182 | public long getP2() { 183 | return p2; 184 | } 185 | 186 | public long getP3() { 187 | return p3; 188 | } 189 | 190 | public long getP4() { 191 | return p4; 192 | } 193 | 194 | public long getP5() { 195 | return p5; 196 | } 197 | 198 | public long getP6() { 199 | return p6; 200 | } 201 | 202 | public long getP7() { 203 | return p7; 204 | } 205 | 206 | public long getP8() { 207 | return p8; 208 | } 209 | 210 | public long getP9() { 211 | return p9; 212 | } 213 | 214 | public long getP10() { 215 | return p10; 216 | } 217 | 218 | public long getP11() { 219 | return p11; 220 | } 221 | 222 | public long getP12() { 223 | return p12; 224 | } 225 | 226 | public long getP13() { 227 | return p13; 228 | } 229 | 230 | public long getP14() { 231 | return p14; 232 | } 233 | 234 | public long getP15() { 235 | return p15; 236 | } 237 | 238 | public long getP16() { 239 | return p16; 240 | } 241 | } 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/LongStoreKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.Externalizable; 20 | import java.io.IOException; 21 | import java.io.ObjectInput; 22 | import java.io.ObjectOutput; 23 | 24 | import com.github.garychenc.filemap.util.Asserts; 25 | 26 | /** 27 | * 28 | * @author Gary CHEN 29 | * 30 | */ 31 | public class LongStoreKey implements StoreKey, Externalizable, Comparable { 32 | 33 | private static final int CLASS_VERSION = 1; 34 | private String uuid; 35 | private long key; 36 | 37 | public LongStoreKey() { 38 | } 39 | 40 | public LongStoreKey(String uuid, long key) { 41 | Asserts.stringNotEmpty("uuid", uuid); 42 | this.uuid = uuid; 43 | this.key = key; 44 | } 45 | 46 | /* (non-Javadoc) 47 | */ 48 | @Override 49 | public String getUuid() { 50 | return uuid; 51 | } 52 | 53 | public void setUUID(String uuid) { 54 | Asserts.stringNotEmpty("uuid", uuid); 55 | this.uuid = uuid; 56 | } 57 | 58 | /* (non-Javadoc) 59 | * @see java.io.Externalizable#readExternal(java.io.ObjectInput) 60 | */ 61 | @Override 62 | public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { 63 | in.readInt(); 64 | this.key = in.readLong(); 65 | this.uuid = in.readUTF(); 66 | } 67 | 68 | /* (non-Javadoc) 69 | * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput) 70 | */ 71 | @Override 72 | public void writeExternal(ObjectOutput out) throws IOException { 73 | out.writeInt(CLASS_VERSION); 74 | out.writeLong(key); 75 | out.writeUTF(uuid); 76 | } 77 | 78 | public long getKey() { 79 | return key; 80 | } 81 | 82 | public void setKey(long key) { 83 | this.key = key; 84 | } 85 | 86 | public String toString() { 87 | StringBuilder description = new StringBuilder(); 88 | description.append("LongStoreKey [uuid = ").append(uuid).append(", key = ").append(key).append("]"); 89 | return description.toString(); 90 | } 91 | 92 | @Override 93 | public int hashCode() { 94 | final int prime = 31; 95 | int result = 1; 96 | result = prime * result + (int) (key ^ (key >>> 32)); 97 | result = prime * result + ((uuid == null) ? 0 : uuid.hashCode()); 98 | return result; 99 | } 100 | 101 | @Override 102 | public boolean equals(Object obj) { 103 | if (this == obj) { 104 | return true; 105 | } 106 | if (obj == null) { 107 | return false; 108 | } 109 | if (!(obj instanceof LongStoreKey)) { 110 | return false; 111 | } 112 | LongStoreKey other = (LongStoreKey) obj; 113 | if (key != other.key) { 114 | return false; 115 | } 116 | if (uuid == null) { 117 | if (other.uuid != null) { 118 | return false; 119 | } 120 | } else if (!uuid.equals(other.uuid)) { 121 | return false; 122 | } 123 | return true; 124 | } 125 | 126 | @Override 127 | public int compareTo(LongStoreKey another) { 128 | if (this.getKey() > another.getKey()) { 129 | return 1; 130 | } else if (this.getKey() < another.getKey()) { 131 | return -1; 132 | } else { 133 | return 0; 134 | } 135 | } 136 | 137 | @Override 138 | public long getVersionNumber() { 139 | return 0; 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/MallocContentStoreSpaceException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class MallocContentStoreSpaceException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = 552483697393310268L; 30 | 31 | /** 32 | * 33 | */ 34 | public MallocContentStoreSpaceException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public MallocContentStoreSpaceException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public MallocContentStoreSpaceException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public MallocContentStoreSpaceException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/ManagedStoreFileMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import static com.github.garychenc.filemap.util.Asserts.*; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.IOException; 24 | import java.util.concurrent.atomic.AtomicLong; 25 | 26 | import com.github.garychenc.filemap.util.IOHelper; 27 | import com.github.garychenc.filemap.util.Utils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | /** 32 | * 33 | * @author Gary CHEN 34 | * 35 | */ 36 | public class ManagedStoreFileMetaData implements Comparable { 37 | 38 | private final static Logger LOGGER = LoggerFactory.getLogger(ManagedStoreFileMetaData.class); 39 | 40 | public final static int MANAGED_FILE_META_BYTES_LENGTH = 16; 41 | public final static long NULL_FILE_SIZE = -1; 42 | public final static long TOTALLY_REMOVED_FILE_SIZE = -16; 43 | public final static byte[] NULL_MANAGED_FILE_META_BYTES; 44 | 45 | static { 46 | ByteArrayOutputStream outputstream = null; 47 | try { 48 | outputstream = new ByteArrayOutputStream(); 49 | IOHelper.write64bit(outputstream, NULL_FILE_SIZE); 50 | IOHelper.write64bit(outputstream, NULL_FILE_SIZE); 51 | outputstream.flush(); 52 | NULL_MANAGED_FILE_META_BYTES = outputstream.toByteArray(); 53 | } catch (Throwable ex) { 54 | throw new IllegalStateException("Write null managed file meta bytes failed.", ex); 55 | } finally { 56 | if (outputstream != null) { 57 | try { outputstream.close(); } catch (IOException exce) { } 58 | } 59 | } 60 | } 61 | 62 | //cache line padding 1 63 | private long p1, p2, p3, p4, p5, p6, p7, p8; 64 | 65 | private final long fileNumber; 66 | private volatile long fileTotalSize; 67 | private long fileUsedSize; 68 | 69 | //cache line padding 2 70 | private long p9, p10, p11, p12, p13, p14, p15, p16; 71 | 72 | public ManagedStoreFileMetaData(long fileNumber, long fileTotalSize, long fileUsedSize) { 73 | longIsPositive("fileNumber", fileNumber); 74 | 75 | this.fileNumber = fileNumber; 76 | this.fileTotalSize = fileTotalSize; 77 | this.fileUsedSize = fileUsedSize; 78 | } 79 | 80 | public ManagedStoreFileMetaData(int fileNumber, long fileTotalSize) { 81 | this(fileNumber, fileTotalSize, 0); 82 | } 83 | 84 | public long getFileNumber() { 85 | return fileNumber; 86 | } 87 | 88 | public long getFileTotalSize() { 89 | return fileTotalSize; 90 | } 91 | 92 | public void setFileTotalSize(long fileTotalSize) { 93 | this.fileTotalSize = fileTotalSize; 94 | } 95 | 96 | public synchronized long getFileUsedSize() { 97 | return fileUsedSize; 98 | } 99 | 100 | public synchronized void resetFileUsedSize() { 101 | fileUsedSize = 0; 102 | } 103 | 104 | public synchronized long getCurrentStartAddress() { 105 | return fileUsedSize; 106 | } 107 | 108 | public synchronized long increaseUsedSize(long delta) { 109 | fileUsedSize = fileUsedSize + delta; 110 | return fileUsedSize; 111 | } 112 | 113 | public long getFileSpareSize() { 114 | return getFileTotalSize() - getFileUsedSize(); 115 | } 116 | 117 | public long getP1() { 118 | return p1; 119 | } 120 | 121 | public long getP2() { 122 | return p2; 123 | } 124 | 125 | public long getP3() { 126 | return p3; 127 | } 128 | 129 | public long getP4() { 130 | return p4; 131 | } 132 | 133 | public long getP5() { 134 | return p5; 135 | } 136 | 137 | public long getP6() { 138 | return p6; 139 | } 140 | 141 | public long getP7() { 142 | return p7; 143 | } 144 | 145 | public long getP8() { 146 | return p8; 147 | } 148 | 149 | public long getP9() { 150 | return p9; 151 | } 152 | 153 | public long getP10() { 154 | return p10; 155 | } 156 | 157 | public long getP11() { 158 | return p11; 159 | } 160 | 161 | public long getP12() { 162 | return p12; 163 | } 164 | 165 | public long getP13() { 166 | return p13; 167 | } 168 | 169 | public long getP14() { 170 | return p14; 171 | } 172 | 173 | public long getP15() { 174 | return p15; 175 | } 176 | 177 | public long getP16() { 178 | return p16; 179 | } 180 | 181 | @Override 182 | public int compareTo(ManagedStoreFileMetaData another) { 183 | if (this.getFileSpareSize() > another.getFileSpareSize()) { 184 | return 1; 185 | } else if (this.getFileSpareSize() < another.getFileSpareSize()) { 186 | return -1; 187 | } else { 188 | return 0; 189 | } 190 | } 191 | 192 | 193 | public static ManagedStoreFileMetaData fromBytes(byte[] bytes, long fileNumber, boolean ignoreErrorEntry) throws IOException { 194 | if (ignoreErrorEntry) { 195 | if (bytes == null || bytes.length != MANAGED_FILE_META_BYTES_LENGTH) { 196 | return new ManagedStoreFileMetaData(fileNumber, TOTALLY_REMOVED_FILE_SIZE, 0); 197 | } 198 | 199 | ByteArrayInputStream inputstream = null; 200 | try { 201 | inputstream = new ByteArrayInputStream(bytes); 202 | long totalSize = IOHelper.read64bit(inputstream); 203 | long usedSize = IOHelper.read64bit(inputstream); 204 | if (totalSize == NULL_FILE_SIZE || usedSize == NULL_FILE_SIZE) { 205 | return null; 206 | } else { 207 | return new ManagedStoreFileMetaData(fileNumber, totalSize, usedSize); 208 | } 209 | } catch (Throwable ex) { 210 | LOGGER.error("Read ManagedStoreFileMetaData failed. fileNumber : " + fileNumber, ex); 211 | return new ManagedStoreFileMetaData(fileNumber, TOTALLY_REMOVED_FILE_SIZE, 0); 212 | } finally { 213 | Utils.closeInputStream(inputstream); 214 | } 215 | } else { 216 | if (bytes == null || bytes.length != MANAGED_FILE_META_BYTES_LENGTH) { 217 | throw new IllegalArgumentException("Bytes length error. Expect length : " + MANAGED_FILE_META_BYTES_LENGTH); 218 | } 219 | 220 | ByteArrayInputStream inputstream = null; 221 | try { 222 | inputstream = new ByteArrayInputStream(bytes); 223 | long totalSize = IOHelper.read64bit(inputstream); 224 | long usedSize = IOHelper.read64bit(inputstream); 225 | if (totalSize == NULL_FILE_SIZE || usedSize == NULL_FILE_SIZE) { 226 | return null; 227 | } else { 228 | return new ManagedStoreFileMetaData(fileNumber, totalSize, usedSize); 229 | } 230 | } finally { 231 | Utils.closeInputStream(inputstream); 232 | } 233 | } 234 | } 235 | 236 | public static byte[] toBytes(ManagedStoreFileMetaData fileMeta) throws IOException { 237 | try (ByteArrayOutputStream outputstream = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 238 | if (fileMeta == null) { 239 | IOHelper.write64bit(outputstream, NULL_FILE_SIZE); 240 | IOHelper.write64bit(outputstream, NULL_FILE_SIZE); 241 | } else { 242 | IOHelper.write64bit(outputstream, fileMeta.getFileTotalSize()); 243 | IOHelper.write64bit(outputstream, fileMeta.getFileUsedSize()); 244 | } 245 | 246 | outputstream.flush(); 247 | return outputstream.toByteArray(); 248 | } 249 | } 250 | 251 | } 252 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/SearchReachedMaxBucketsNumberException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class SearchReachedMaxBucketsNumberException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = -4990076830337679587L; 30 | 31 | /** 32 | * 33 | */ 34 | public SearchReachedMaxBucketsNumberException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public SearchReachedMaxBucketsNumberException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public SearchReachedMaxBucketsNumberException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public SearchReachedMaxBucketsNumberException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/StoreKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface StoreKey { 25 | 26 | public String getUuid(); 27 | public long getVersionNumber(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/StoreValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface StoreValue { 25 | 26 | public StoreKey getOriginalStoreKey(); 27 | public void setOriginalStoreKey(StoreKey key); 28 | 29 | public long getVersionNumber(); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/store/filestore/StringStoreKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.store.filestore; 18 | 19 | import java.io.Externalizable; 20 | import java.io.IOException; 21 | import java.io.ObjectInput; 22 | import java.io.ObjectOutput; 23 | 24 | import com.github.garychenc.filemap.util.Asserts; 25 | 26 | /** 27 | * 28 | * @author Gary CHEN 29 | * 30 | */ 31 | public class StringStoreKey implements StoreKey, Externalizable { 32 | 33 | private static final int CLASS_VERSION = 1; 34 | private String uuid; 35 | private String key; 36 | 37 | public StringStoreKey() { 38 | } 39 | 40 | public StringStoreKey(String uuid, String key) { 41 | Asserts.stringNotEmpty("uuid", uuid); 42 | Asserts.stringNotEmpty("key", key); 43 | this.uuid = uuid; 44 | this.key = key; 45 | } 46 | 47 | /* (non-Javadoc) 48 | */ 49 | @Override 50 | public String getUuid() { 51 | return uuid; 52 | } 53 | 54 | public void setUUID(String uuid) { 55 | Asserts.stringNotEmpty("uuid", uuid); 56 | this.uuid = uuid; 57 | } 58 | 59 | /* (non-Javadoc) 60 | * @see java.io.Externalizable#readExternal(java.io.ObjectInput) 61 | */ 62 | @Override 63 | public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { 64 | in.readInt(); 65 | this.key = in.readUTF(); 66 | this.uuid = in.readUTF(); 67 | } 68 | 69 | /* (non-Javadoc) 70 | * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput) 71 | */ 72 | @Override 73 | public void writeExternal(ObjectOutput out) throws IOException { 74 | out.writeInt(CLASS_VERSION); 75 | out.writeUTF(key); 76 | out.writeUTF(uuid); 77 | } 78 | 79 | public String getKey() { 80 | return key; 81 | } 82 | 83 | public void setKey(String key) { 84 | Asserts.stringNotEmpty("key", key); 85 | this.key = key; 86 | } 87 | 88 | @Override 89 | public int hashCode() { 90 | final int prime = 31; 91 | int result = 1; 92 | result = prime * result + ((key == null) ? 0 : key.hashCode()); 93 | result = prime * result + ((uuid == null) ? 0 : uuid.hashCode()); 94 | return result; 95 | } 96 | 97 | @Override 98 | public boolean equals(Object obj) { 99 | if (this == obj) { 100 | return true; 101 | } 102 | if (obj == null) { 103 | return false; 104 | } 105 | if (!(obj instanceof StringStoreKey)) { 106 | return false; 107 | } 108 | StringStoreKey other = (StringStoreKey) obj; 109 | if (key == null) { 110 | if (other.key != null) { 111 | return false; 112 | } 113 | } else if (!key.equals(other.key)) { 114 | return false; 115 | } 116 | if (uuid == null) { 117 | if (other.uuid != null) { 118 | return false; 119 | } 120 | } else if (!uuid.equals(other.uuid)) { 121 | return false; 122 | } 123 | return true; 124 | } 125 | 126 | public String toString() { 127 | StringBuilder description = new StringBuilder(); 128 | description.append("StringStoreKey [uuid = ").append(uuid).append(", key = ").append(key).append("]"); 129 | return description.toString(); 130 | } 131 | 132 | @Override 133 | public long getVersionNumber() { 134 | return 0; 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/Asserts.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public final class Asserts { 25 | 26 | private Asserts() { 27 | } 28 | 29 | public static final void notNull(String name, Object value) { 30 | if (value == null) { 31 | throw new IllegalArgumentException("Parameter '" + name + "' can not be null."); 32 | } 33 | } 34 | 35 | public static final void stringNotEmpty(String name, String value) { 36 | notNull(name, value); 37 | if ("".equals(value.trim())) { 38 | throw new IllegalArgumentException("String parameter '" + name + "' can not be empty."); 39 | } 40 | } 41 | 42 | public static final void longIsPositive(String name, long value) { 43 | if (value < 0) { 44 | throw new IllegalArgumentException("Long parameter '" + name + "' must be a positive number."); 45 | } 46 | } 47 | 48 | public static final void longGtZero(String name, long value) { 49 | if (value <= 0) { 50 | throw new IllegalArgumentException("Long parameter '" + name + "' must be greater than zero."); 51 | } 52 | } 53 | 54 | public static final void integerIsPositive(String name, int value) { 55 | if (value < 0) { 56 | throw new IllegalArgumentException("Integer parameter '" + name + "' must be a positive number."); 57 | } 58 | } 59 | 60 | public static final void doubleIsPositive(String name, double value) { 61 | if (value < 0) { 62 | throw new IllegalArgumentException("Double parameter '" + name + "' must be a positive number."); 63 | } 64 | } 65 | 66 | public static final void integerGtZero(String name, int value) { 67 | if (value <= 0) { 68 | throw new IllegalArgumentException("Integer parameter '" + name + "' must be greater than zero."); 69 | } 70 | } 71 | 72 | public static void isTrue(boolean expression, String message) { 73 | if (!expression) { 74 | throw new IllegalArgumentException(message); 75 | } 76 | } 77 | 78 | public static void isTrue(boolean expression) { 79 | isTrue(expression, "[Assertion failed] - this expression must be true"); 80 | } 81 | 82 | public static void isNull(Object object, String message) { 83 | if (object != null) { 84 | throw new IllegalArgumentException(message); 85 | } 86 | } 87 | 88 | public static void isNull(Object object) { 89 | isNull(object, "[Assertion failed] - the object argument must be null"); 90 | } 91 | } 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/ByteArrayObjectInput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.IOException; 21 | import java.io.ObjectInput; 22 | 23 | /** 24 | * 25 | * @author Gary CHEN 26 | * 27 | */ 28 | public class ByteArrayObjectInput implements ObjectInput { 29 | 30 | private final ByteArrayInputStream inputstream; 31 | private final boolean delegate; 32 | 33 | public ByteArrayObjectInput(byte[] in) { 34 | Asserts.notNull("in", in); 35 | this.inputstream = new ByteArrayInputStream(in); 36 | this.delegate = false; 37 | } 38 | 39 | public ByteArrayObjectInput(ByteArrayInputStream inputstream, boolean delegate) { 40 | Asserts.notNull("inputstream", inputstream); 41 | this.inputstream = inputstream; 42 | this.delegate = delegate; 43 | } 44 | 45 | /* (non-Javadoc) 46 | * @see java.io.ObjectInput#available() 47 | */ 48 | @Override 49 | public int available() throws IOException { 50 | return inputstream.available(); 51 | } 52 | 53 | /* (non-Javadoc) 54 | * @see java.io.ObjectInput#close() 55 | */ 56 | @Override 57 | public void close() throws IOException { 58 | if (!delegate) { 59 | inputstream.close(); 60 | } 61 | } 62 | 63 | /* (non-Javadoc) 64 | * @see java.io.ObjectInput#read() 65 | */ 66 | @Override 67 | public int read() throws IOException { 68 | return IOHelper.read8bit(inputstream); 69 | } 70 | 71 | /* (non-Javadoc) 72 | * @see java.io.ObjectInput#read(byte[]) 73 | */ 74 | @Override 75 | public int read(byte[] b) throws IOException { 76 | byte[] bytes = IOHelper.readByteArray32(inputstream); 77 | if (b.length != bytes.length) { 78 | throw new IOException("the length of input argument bytes error. Expected: " + bytes.length + ",Actual: " + b.length + "."); 79 | } 80 | 81 | System.arraycopy(bytes, 0, b, 0, bytes.length); 82 | return bytes.length; 83 | } 84 | 85 | /* (non-Javadoc) 86 | * @see java.io.ObjectInput#read(byte[], int, int) 87 | */ 88 | @Override 89 | public int read(byte[] b, int off, int len) throws IOException { 90 | return inputstream.read(b, off, len); 91 | } 92 | 93 | /* (non-Javadoc) 94 | * @see java.io.ObjectInput#readObject() 95 | */ 96 | @Override 97 | public Object readObject() throws ClassNotFoundException, IOException { 98 | return IOHelper.readObject(inputstream); 99 | } 100 | 101 | /* (non-Javadoc) 102 | * @see java.io.ObjectInput#skip(long) 103 | */ 104 | @Override 105 | public long skip(long n) throws IOException { 106 | return inputstream.skip(n); 107 | } 108 | 109 | /* (non-Javadoc) 110 | * @see java.io.DataInput#readBoolean() 111 | */ 112 | @Override 113 | public boolean readBoolean() throws IOException { 114 | return IOHelper.readBoolean(inputstream); 115 | } 116 | 117 | /* (non-Javadoc) 118 | * @see java.io.DataInput#readByte() 119 | */ 120 | @Override 121 | public byte readByte() throws IOException { 122 | return (byte)IOHelper.read8bit(inputstream); 123 | } 124 | 125 | /* (non-Javadoc) 126 | * @see java.io.DataInput#readChar() 127 | */ 128 | @Override 129 | public char readChar() throws IOException { 130 | return (char)IOHelper.read16bit(inputstream); 131 | } 132 | 133 | /* (non-Javadoc) 134 | * @see java.io.DataInput#readDouble() 135 | */ 136 | @Override 137 | public double readDouble() throws IOException { 138 | return IOHelper.readDouble(inputstream); 139 | } 140 | 141 | /* (non-Javadoc) 142 | * @see java.io.DataInput#readFloat() 143 | */ 144 | @Override 145 | public float readFloat() throws IOException { 146 | return IOHelper.readFloat(inputstream); 147 | } 148 | 149 | /* (non-Javadoc) 150 | * @see java.io.DataInput#readFully(byte[]) 151 | */ 152 | @Override 153 | public void readFully(byte[] b) throws IOException { 154 | throw new UnsupportedOperationException(); 155 | } 156 | 157 | /* (non-Javadoc) 158 | * @see java.io.DataInput#readFully(byte[], int, int) 159 | */ 160 | @Override 161 | public void readFully(byte[] b, int off, int len) throws IOException { 162 | throw new UnsupportedOperationException(); 163 | } 164 | 165 | /* (non-Javadoc) 166 | * @see java.io.DataInput#readInt() 167 | */ 168 | @Override 169 | public int readInt() throws IOException { 170 | return IOHelper.read32bit(inputstream); 171 | } 172 | 173 | /* (non-Javadoc) 174 | * @see java.io.DataInput#readLine() 175 | */ 176 | @Override 177 | public String readLine() throws IOException { 178 | return IOHelper.readUTF32(inputstream); 179 | } 180 | 181 | /* (non-Javadoc) 182 | * @see java.io.DataInput#readLong() 183 | */ 184 | @Override 185 | public long readLong() throws IOException { 186 | return IOHelper.read64bit(inputstream); 187 | } 188 | 189 | /* (non-Javadoc) 190 | * @see java.io.DataInput#readShort() 191 | */ 192 | @Override 193 | public short readShort() throws IOException { 194 | return (short)IOHelper.read16bit(inputstream); 195 | } 196 | 197 | /* (non-Javadoc) 198 | * @see java.io.DataInput#readUTF() 199 | */ 200 | @Override 201 | public String readUTF() throws IOException { 202 | return IOHelper.readUTF32(inputstream); 203 | } 204 | 205 | /* (non-Javadoc) 206 | * @see java.io.DataInput#readUnsignedByte() 207 | */ 208 | @Override 209 | public int readUnsignedByte() throws IOException { 210 | return IOHelper.read16bit(inputstream); 211 | } 212 | 213 | /* (non-Javadoc) 214 | * @see java.io.DataInput#readUnsignedShort() 215 | */ 216 | @Override 217 | public int readUnsignedShort() throws IOException { 218 | return IOHelper.read16bit(inputstream); 219 | } 220 | 221 | /* (non-Javadoc) 222 | * @see java.io.DataInput#skipBytes(int) 223 | */ 224 | @Override 225 | public int skipBytes(int n) throws IOException { 226 | return (int)inputstream.skip(n); 227 | } 228 | 229 | } 230 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/ByteArrayObjectOutput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | import java.io.ByteArrayOutputStream; 20 | import java.io.IOException; 21 | import java.io.ObjectOutput; 22 | 23 | /** 24 | * 25 | * @author Gary CHEN 26 | * 27 | */ 28 | public class ByteArrayObjectOutput implements ObjectOutput { 29 | 30 | private final ByteArrayOutputStream outputStream; 31 | private final boolean delegate; 32 | 33 | public ByteArrayObjectOutput(ByteArrayOutputStream outputStream, boolean delegate) { 34 | Asserts.notNull("ByteArrayOutputStream", outputStream); 35 | this.outputStream = outputStream; 36 | this.delegate = delegate; 37 | } 38 | 39 | /* (non-Javadoc) 40 | * @see java.io.ObjectOutput#close() 41 | */ 42 | @Override 43 | public void close() throws IOException { 44 | if (!delegate) { 45 | outputStream.close(); 46 | } 47 | } 48 | 49 | /* (non-Javadoc) 50 | * @see java.io.ObjectOutput#flush() 51 | */ 52 | @Override 53 | public void flush() throws IOException { 54 | outputStream.flush(); 55 | } 56 | 57 | /* (non-Javadoc) 58 | * @see java.io.ObjectOutput#write(int) 59 | */ 60 | @Override 61 | public void write(int b) throws IOException { 62 | IOHelper.write8bit(outputStream, b); 63 | } 64 | 65 | /* (non-Javadoc) 66 | * @see java.io.ObjectOutput#write(byte[]) 67 | */ 68 | @Override 69 | public void write(byte[] b) throws IOException { 70 | IOHelper.writeByteArray32(outputStream, b); 71 | } 72 | 73 | /* (non-Javadoc) 74 | * @see java.io.ObjectOutput#write(byte[], int, int) 75 | */ 76 | @Override 77 | public void write(byte[] b, int off, int len) throws IOException { 78 | byte[] copy = Utils.copyOfRange(b, off, off + len); 79 | IOHelper.writeByteArray32(outputStream, copy); 80 | 81 | } 82 | 83 | /* (non-Javadoc) 84 | * @see java.io.ObjectOutput#writeObject(java.lang.Object) 85 | */ 86 | @Override 87 | public void writeObject(Object obj) throws IOException { 88 | IOHelper.writeObject(outputStream, obj); 89 | } 90 | 91 | /* (non-Javadoc) 92 | * @see java.io.DataOutput#writeBoolean(boolean) 93 | */ 94 | @Override 95 | public void writeBoolean(boolean v) throws IOException { 96 | IOHelper.writeBoolean(outputStream, v); 97 | } 98 | 99 | /* (non-Javadoc) 100 | * @see java.io.DataOutput#writeByte(int) 101 | */ 102 | @Override 103 | public void writeByte(int v) throws IOException { 104 | IOHelper.write8bit(outputStream, v); 105 | } 106 | 107 | /* (non-Javadoc) 108 | * @see java.io.DataOutput#writeBytes(java.lang.String) 109 | */ 110 | @Override 111 | public void writeBytes(String s) throws IOException { 112 | IOHelper.writeUTF32(outputStream, s); 113 | } 114 | 115 | /* (non-Javadoc) 116 | * @see java.io.DataOutput#writeChar(int) 117 | */ 118 | @Override 119 | public void writeChar(int v) throws IOException { 120 | IOHelper.write16bit(outputStream, v); 121 | } 122 | 123 | /* (non-Javadoc) 124 | * @see java.io.DataOutput#writeChars(java.lang.String) 125 | */ 126 | @Override 127 | public void writeChars(String s) throws IOException { 128 | IOHelper.writeUTF32(outputStream, s); 129 | } 130 | 131 | /* (non-Javadoc) 132 | * @see java.io.DataOutput#writeDouble(double) 133 | */ 134 | @Override 135 | public void writeDouble(double v) throws IOException { 136 | IOHelper.writeDouble(outputStream, v); 137 | } 138 | 139 | /* (non-Javadoc) 140 | * @see java.io.DataOutput#writeFloat(float) 141 | */ 142 | @Override 143 | public void writeFloat(float v) throws IOException { 144 | IOHelper.writeFloat(outputStream, v); 145 | } 146 | 147 | /* (non-Javadoc) 148 | * @see java.io.DataOutput#writeInt(int) 149 | */ 150 | @Override 151 | public void writeInt(int v) throws IOException { 152 | IOHelper.write32bit(outputStream, v); 153 | } 154 | 155 | /* (non-Javadoc) 156 | * @see java.io.DataOutput#writeLong(long) 157 | */ 158 | @Override 159 | public void writeLong(long v) throws IOException { 160 | IOHelper.write64bit(outputStream, v); 161 | } 162 | 163 | /* (non-Javadoc) 164 | * @see java.io.DataOutput#writeShort(int) 165 | */ 166 | @Override 167 | public void writeShort(int v) throws IOException { 168 | IOHelper.write16bit(outputStream, v); 169 | } 170 | 171 | /* (non-Javadoc) 172 | * @see java.io.DataOutput#writeUTF(java.lang.String) 173 | */ 174 | @Override 175 | public void writeUTF(String s) throws IOException { 176 | IOHelper.writeUTF32(outputStream, s); 177 | } 178 | 179 | } 180 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/DeleteDirectoryContentsFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class DeleteDirectoryContentsFailedException extends Exception { 25 | 26 | /** 27 | * 28 | */ 29 | private static final long serialVersionUID = -3121073428054873353L; 30 | 31 | /** 32 | * 33 | */ 34 | public DeleteDirectoryContentsFailedException() { 35 | } 36 | 37 | /** 38 | * @param message 39 | */ 40 | public DeleteDirectoryContentsFailedException(String message) { 41 | super(message); 42 | } 43 | 44 | /** 45 | * @param cause 46 | */ 47 | public DeleteDirectoryContentsFailedException(Throwable cause) { 48 | super(cause); 49 | } 50 | 51 | /** 52 | * @param message 53 | * @param cause 54 | */ 55 | public DeleteDirectoryContentsFailedException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/HexDumper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public final class HexDumper { 25 | 26 | /** 27 | * The high digits lookup table. 28 | */ 29 | private static final byte[] highDigits; 30 | 31 | /** 32 | * The low digits lookup table. 33 | */ 34 | private static final byte[] lowDigits; 35 | 36 | /** 37 | * Initialize lookup tables. 38 | */ 39 | static { 40 | final byte[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; 41 | 42 | int i; 43 | byte[] high = new byte[256]; 44 | byte[] low = new byte[256]; 45 | 46 | for (i = 0; i < 256; i++) { 47 | high[i] = digits[i >>> 4]; 48 | low[i] = digits[i & 0x0F]; 49 | } 50 | 51 | highDigits = high; 52 | lowDigits = low; 53 | } 54 | 55 | private HexDumper() { 56 | } 57 | 58 | public static final String getHexdump(byte[] in, int lengthLimit) { 59 | if (lengthLimit == 0) { 60 | throw new IllegalArgumentException("lengthLimit: " + lengthLimit + " (expected: 1+)"); 61 | } 62 | 63 | boolean truncate = in.length > lengthLimit; 64 | int size; 65 | if (truncate) { 66 | size = lengthLimit; 67 | } else { 68 | size = in.length; 69 | } 70 | 71 | if (size == 0) { 72 | return "empty"; 73 | } 74 | 75 | StringBuilder out = new StringBuilder(size * 3 + 3); 76 | 77 | int index = 0; 78 | // fill the first 79 | int byteValue = in[index++] & 0xFF; 80 | out.append((char) highDigits[byteValue]); 81 | out.append((char) lowDigits[byteValue]); 82 | size--; 83 | 84 | // and the others, too 85 | for (; size > 0; size--) { 86 | out.append(' '); 87 | byteValue = in[index++] & 0xFF; 88 | out.append((char) highDigits[byteValue]); 89 | out.append((char) lowDigits[byteValue]); 90 | } 91 | 92 | if (truncate) { 93 | out.append("..."); 94 | } 95 | 96 | return out.toString(); 97 | } 98 | 99 | public static final String getHexdump(byte[] in) { 100 | return getHexdump(in, Integer.MAX_VALUE); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/ObjectCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface ObjectCache { 25 | 26 | public V get(K key); 27 | public V put(K key, V value); 28 | public V remove(K key); 29 | public void clear(); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/ObjectCacheFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public interface ObjectCacheFactory { 25 | 26 | public ObjectCache createObjectCache(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/SerializationUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | import java.io.ByteArrayInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | import java.io.ObjectInputStream; 23 | import java.io.ObjectOutputStream; 24 | import java.io.Serializable; 25 | 26 | /** 27 | * 28 | * @author Gary CHEN 29 | * 30 | */ 31 | public final class SerializationUtils { 32 | 33 | private SerializationUtils() { 34 | } 35 | 36 | /** 37 | * Serialize the given object to a byte array. 38 | * 39 | * @param object 40 | * the object to serialize 41 | * @return an array of bytes representing the object in a portable fashion 42 | */ 43 | public final static byte[] serialize(Object object) { 44 | if (object == null) { 45 | return null; 46 | } 47 | 48 | try (ByteArrayOutputStream baos = new ByteArrayOutputStream(IOHelper.BUFFER_DEFAULT_ALLOCATED_SIZE)) { 49 | ObjectOutputStream oos = new ObjectOutputStream(baos); 50 | oos.writeObject(object); 51 | oos.flush(); 52 | return baos.toByteArray(); 53 | } catch (IOException ex) { 54 | throw new IllegalArgumentException("Failed to serialize object of type: " + object.getClass(), ex); 55 | } 56 | } 57 | 58 | /** 59 | * Deserialize the byte array into an object. 60 | * 61 | * @param bytes 62 | * a serialized object 63 | * @return the result of deserializing the bytes 64 | */ 65 | public final static Object deserialize(byte[] bytes) { 66 | if (bytes == null) { 67 | return null; 68 | } 69 | 70 | try { 71 | ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); 72 | return ois.readObject(); 73 | } catch (IOException ex) { 74 | throw new IllegalArgumentException("Failed to deserialize object", ex); 75 | } catch (ClassNotFoundException ex) { 76 | throw new IllegalStateException("Failed to deserialize object type", ex); 77 | } 78 | } 79 | 80 | // Clone 81 | //----------------------------------------------------------------------- 82 | /** 83 | *

Deep clone an Object using serialization.

84 | * 85 | *

This is many times slower than writing clone methods by hand 86 | * on all objects in your object graph. However, for complex object 87 | * graphs, or for those that don't support deep cloning this can 88 | * be a simple alternative implementation. Of course all the objects 89 | * must be Serializable.

90 | * 91 | * @param object the Serializable object to clone 92 | * @return the cloned object 93 | */ 94 | public final static Object clone(Serializable object) { 95 | return deserialize(serialize(object)); 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/SimpleObjectCacheFactoryImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | /** 20 | * 21 | * @author Gary CHEN 22 | * 23 | */ 24 | public class SimpleObjectCacheFactoryImpl implements ObjectCacheFactory { 25 | 26 | private final int cacheSize; 27 | 28 | public SimpleObjectCacheFactoryImpl(int cacheSize) { 29 | assert cacheSize > 0; 30 | this.cacheSize = cacheSize; 31 | } 32 | 33 | /* (non-Javadoc) 34 | * @see scnu.cs.spider.cache.ObjectCacheFactory#createObjectCache() 35 | */ 36 | public ObjectCache createObjectCache() { 37 | return new SimpleObjectCacheImpl(cacheSize); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/SimpleObjectCacheImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | import java.lang.ref.SoftReference; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import java.util.concurrent.ConcurrentMap; 22 | 23 | /** 24 | * 25 | * @author Gary CHEN 26 | * 27 | */ 28 | class SimpleObjectCacheImpl implements ObjectCache { 29 | 30 | private final int cacheSize; 31 | private final ConcurrentMap> cache; 32 | 33 | SimpleObjectCacheImpl(int cacheSize) { 34 | assert cacheSize > 0; 35 | this.cacheSize = cacheSize; 36 | this.cache = new ConcurrentHashMap>(); 37 | } 38 | 39 | /* (non-Javadoc) 40 | * @see scnu.cs.spider.cache.ObjectCache#clear() 41 | */ 42 | public void clear() { 43 | cache.clear(); 44 | } 45 | 46 | private void clearCacheIfNecessary() { 47 | int size = cache.size(); 48 | if (size >= cacheSize) { 49 | cache.clear(); 50 | } 51 | } 52 | 53 | /* (non-Javadoc) 54 | * @see scnu.cs.spider.cache.ObjectCache#get(java.lang.Object) 55 | */ 56 | public V get(K key) { 57 | try { 58 | SoftReference valueRef = cache.get(key); 59 | if (valueRef == null) { 60 | return null; 61 | } 62 | else { 63 | V value = valueRef.get(); 64 | if (value == null && cache.containsKey(key)) { 65 | cache.remove(key); 66 | } 67 | 68 | return value; 69 | } 70 | } finally { 71 | clearCacheIfNecessary(); 72 | } 73 | } 74 | 75 | /* (non-Javadoc) 76 | * @see scnu.cs.spider.cache.ObjectCache#put(java.lang.Object, java.lang.Object) 77 | */ 78 | public V put(K key, V value) { 79 | clearCacheIfNecessary(); 80 | SoftReference valueRef = cache.put(key, new SoftReference(value)); 81 | if (valueRef == null) { 82 | return null; 83 | } 84 | else { 85 | return valueRef.get(); 86 | } 87 | } 88 | 89 | /* (non-Javadoc) 90 | * @see scnu.cs.spider.cache.ObjectCache#remove(java.lang.Object) 91 | */ 92 | public V remove(K key) { 93 | SoftReference valueRef = cache.remove(key); 94 | if (valueRef == null) { 95 | return null; 96 | } 97 | else { 98 | return valueRef.get(); 99 | } 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /code/src/main/java/com/github/garychenc/filemap/util/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.github.garychenc.filemap.util; 18 | 19 | import java.io.BufferedInputStream; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.File; 22 | import java.io.FileFilter; 23 | import java.io.FileInputStream; 24 | import java.io.IOException; 25 | import java.io.InputStream; 26 | import java.io.OutputStream; 27 | import java.io.RandomAccessFile; 28 | import java.io.Reader; 29 | import java.io.Writer; 30 | import java.net.HttpURLConnection; 31 | import java.net.URL; 32 | import java.net.URLConnection; 33 | import java.util.Set; 34 | 35 | /** 36 | * 37 | * @author Gary CHEN 38 | * 39 | */ 40 | public final class Utils { 41 | 42 | private Utils() { 43 | } 44 | 45 | private static final String[] ZERO_ARRAY = new String[0]; 46 | public static final String DEFAULT_SPLITTER = ","; 47 | public static final String[] splitString2StringArray(String target) { 48 | return splitString2StringArray(target, DEFAULT_SPLITTER); 49 | } 50 | 51 | public static final String[] splitString2StringArray(String target, String spliter) { 52 | if (target == null || target.trim().length() == 0 || 53 | spliter == null || spliter.trim().length() == 0) { 54 | return ZERO_ARRAY; 55 | } else { 56 | String[] array = target.split(spliter); 57 | if (array == null) { 58 | array = ZERO_ARRAY; 59 | } 60 | 61 | for (int i = 0; i < array.length; ++i) { 62 | if (array[i] != null) { 63 | array[i] = array[i].trim(); 64 | } 65 | } 66 | 67 | return array; 68 | } 69 | } 70 | 71 | public static final String joinStringArray2String(Object[] target, String spliter) { 72 | if (target == null || target.length == 0) { 73 | return ""; 74 | } 75 | 76 | if (spliter == null || spliter.trim().length() == 0) { 77 | spliter = DEFAULT_SPLITTER; 78 | } 79 | 80 | StringBuilder ret = new StringBuilder(); 81 | for (Object obj : target) { 82 | if (obj != null) { 83 | ret.append(obj.toString()).append(spliter); 84 | } 85 | } 86 | 87 | ret.delete(ret.length() - spliter.length(), ret.length()); 88 | return ret.toString(); 89 | } 90 | 91 | public static final String joinStringArray2String(Object[] target) { 92 | return joinStringArray2String(target, DEFAULT_SPLITTER); 93 | } 94 | 95 | public static final String joinStringSet2String(Set target, String spliter) { 96 | if (target == null) { 97 | return ""; 98 | } 99 | 100 | return joinStringArray2String(target.toArray(), spliter); 101 | } 102 | 103 | public static final String joinStringSet2String(Set target) { 104 | return joinStringSet2String(target, DEFAULT_SPLITTER); 105 | } 106 | 107 | public static byte[] readFileContentAsBytes(String path) throws IOException { 108 | return readFileContentAsBytes(path, 1024 * 8); 109 | } 110 | 111 | public static final byte[] readFileContentAsBytes(String path, int bufferSize) throws IOException { 112 | File p = new File(path); 113 | return readFileContentAsBytes(p, bufferSize); 114 | } 115 | 116 | public static final byte[] readFileContentAsBytes(File path) throws IOException { 117 | return readFileContentAsBytes(path, 1024 * 8); 118 | } 119 | 120 | public static final byte[] readFileContentAsBytes(File path, int bufferSize) throws IOException { 121 | if (path == null) { 122 | throw new IllegalArgumentException("File path to be read can not be null."); 123 | } 124 | 125 | if (!path.exists()) { 126 | throw new IllegalArgumentException("The path is not existent in file system. Path: " + path.getAbsolutePath() + "."); 127 | } 128 | 129 | if (!path.isFile()) { 130 | throw new IllegalArgumentException("The path is not a file. Path: " + path.getAbsolutePath() + "."); 131 | } 132 | 133 | if (bufferSize < 1024) { 134 | bufferSize = 1024; 135 | } 136 | 137 | FileInputStream fileInput = null; 138 | BufferedInputStream bufferedInput = null; 139 | ByteArrayOutputStream bytesOutput = null; 140 | try { 141 | fileInput = new FileInputStream(path); 142 | bufferedInput = new BufferedInputStream(fileInput, 5 * bufferSize); 143 | bytesOutput = new ByteArrayOutputStream(5 * bufferSize); 144 | byte[] buffer = new byte[bufferSize]; 145 | int length = -1; 146 | while ((length = bufferedInput.read(buffer)) != -1) { 147 | bytesOutput.write(buffer, 0, length); 148 | } 149 | bytesOutput.flush(); 150 | return bytesOutput.toByteArray(); 151 | } finally { 152 | closeOutputStream(bytesOutput); 153 | closeInputStream(bufferedInput); 154 | closeInputStream(fileInput); 155 | } 156 | } 157 | 158 | public static final File getOrCreateDir(String path) { 159 | Asserts.stringNotEmpty("path", path); 160 | 161 | File dir = new File(path); 162 | return getOrCreateDir(dir); 163 | } 164 | 165 | public static final File getOrCreateDir(String parentPath, String childPath) { 166 | Asserts.stringNotEmpty("parentPath", parentPath); 167 | Asserts.stringNotEmpty("childPath", childPath); 168 | File parentFile = getOrCreateDir(parentPath); 169 | File childFile = new File(parentFile, childPath); 170 | return getOrCreateDir(childFile); 171 | } 172 | 173 | public static final File getOrCreateDir(File dir) { 174 | Asserts.notNull("dir", dir); 175 | 176 | if (!dir.exists() && !dir.mkdirs()) { 177 | throw new IllegalStateException("Carete directories for path: [" + dir.getAbsolutePath() + "] failed."); 178 | } 179 | 180 | if (!dir.isDirectory()) { 181 | throw new IllegalStateException("The path: [" + dir.getAbsolutePath() + "] is not a directory."); 182 | } 183 | 184 | return dir; 185 | } 186 | 187 | public static final ClassLoader retrieveClassLoader() { 188 | ClassLoader cl = Utils.class.getClassLoader(); 189 | 190 | if (cl == null) { 191 | throw new IllegalStateException("Can not get a ClassLoader to load classes."); 192 | } 193 | 194 | return cl; 195 | } 196 | 197 | public static final void closeRandomAccessFile(RandomAccessFile accessFile) { 198 | if (accessFile != null) { 199 | try { accessFile.close(); } catch (Throwable ex) {} 200 | } 201 | } 202 | 203 | public static final void closeOutputStream(OutputStream out) { 204 | if (out != null) { 205 | try { out.close(); } catch (Throwable ex) {} 206 | } 207 | } 208 | 209 | public static final void closeInputStream(InputStream in) { 210 | if (in != null) { 211 | try { in.close(); } catch (Throwable ex) {} 212 | } 213 | } 214 | 215 | public static final void closeReader(Reader reader) { 216 | if (reader != null) { 217 | try { reader.close(); } catch (Throwable ex) {} 218 | } 219 | } 220 | 221 | public static final void closeWriter(Writer writer) { 222 | if (writer != null) { 223 | try { writer.close(); } catch (Throwable ex) {} 224 | } 225 | } 226 | 227 | public static final void disconnectUrlConnection(URLConnection conn) { 228 | if (conn instanceof HttpURLConnection) { 229 | try { ((HttpURLConnection)conn).disconnect(); } catch (Throwable ex) {} 230 | } 231 | } 232 | 233 | public static final boolean isUrlHttpCompatible(URL url) { 234 | assert url != null; 235 | return "http".equalsIgnoreCase(url.getProtocol().trim()); 236 | } 237 | 238 | public static final void closeIOResource(InputStream in, OutputStream out, 239 | Reader reader, Writer writer) { 240 | closeInputStream(in); 241 | closeOutputStream(out); 242 | closeReader(reader); 243 | closeWriter(writer); 244 | } 245 | 246 | public static final void closeIOResource(InputStream in, OutputStream out, 247 | Reader reader) { 248 | closeInputStream(in); 249 | closeOutputStream(out); 250 | closeReader(reader); 251 | } 252 | 253 | public static final void closeIOResource(InputStream in, OutputStream out) { 254 | closeInputStream(in); 255 | closeOutputStream(out); 256 | } 257 | 258 | public static final void closeIOResource(Reader reader, Writer writer) { 259 | closeReader(reader); 260 | closeWriter(writer); 261 | } 262 | 263 | public static final void closeIOResource(InputStream in, OutputStream out, 264 | Writer writer) { 265 | closeInputStream(in); 266 | closeOutputStream(out); 267 | closeWriter(writer); 268 | } 269 | 270 | public static final void closeIOResource(InputStream in, Reader reader) { 271 | closeInputStream(in); 272 | closeReader(reader); 273 | } 274 | 275 | public static final void closeIOResource(OutputStream out, Writer writer) { 276 | closeOutputStream(out); 277 | closeWriter(writer); 278 | } 279 | 280 | public static final void closeIOResource(InputStream in, Reader reader, Writer writer) { 281 | closeInputStream(in); 282 | closeReader(reader); 283 | closeWriter(writer); 284 | } 285 | 286 | public static final void closeIOResource(OutputStream out, Reader reader, Writer writer) { 287 | closeOutputStream(out); 288 | closeReader(reader); 289 | closeWriter(writer); 290 | } 291 | 292 | public static final void closeIOResource(InputStream in, Writer writer) { 293 | closeInputStream(in); 294 | closeWriter(writer); 295 | } 296 | 297 | public static final void closeIOResource(OutputStream out, Reader reader) { 298 | closeOutputStream(out); 299 | closeReader(reader); 300 | } 301 | 302 | public static final void deleteTheWholeDirectory(File dir) throws DeleteDirectoryContentsFailedException { 303 | deleteTheWholeDirectory(dir, null); 304 | } 305 | 306 | public static final void deleteTheWholeDirectory(File dir, FileFilter filter) throws DeleteDirectoryContentsFailedException { 307 | Asserts.notNull("directory", dir); 308 | 309 | if (!dir.isDirectory()) { 310 | throw new IllegalArgumentException("Not a directory."); 311 | } 312 | 313 | File[] files = dir.listFiles(filter); 314 | if (files == null || files.length == 0) { 315 | if (!dir.delete()) { 316 | throw new DeleteDirectoryContentsFailedException("Path: " + dir.getAbsolutePath()); 317 | } 318 | } else { 319 | for (File file : files) { 320 | if (file != null) { 321 | if (file.isDirectory()) { 322 | deleteTheWholeDirectory(file, filter); 323 | } else { 324 | if (!file.delete()) { 325 | throw new DeleteDirectoryContentsFailedException("Path: " + file.getAbsolutePath()); 326 | } 327 | } 328 | } 329 | } 330 | 331 | if (!dir.delete()) { 332 | throw new DeleteDirectoryContentsFailedException("Path: " + dir.getAbsolutePath()); 333 | } 334 | } 335 | } 336 | 337 | /** 338 | * Copies the specified range of the specified array into a new array. 339 | * The initial index of the range (from) must lie between zero 340 | * and original.length, inclusive. The value at 341 | * original[from] is placed into the initial element of the copy 342 | * (unless from == original.length or from == to). 343 | * Values from subsequent elements in the original array are placed into 344 | * subsequent elements in the copy. The final index of the range 345 | * (to), which must be greater than or equal to from, 346 | * may be greater than original.length, in which case 347 | * (byte)0 is placed in all elements of the copy whose index is 348 | * greater than or equal to original.length - from. The length 349 | * of the returned array will be to - from. 350 | * 351 | * @param original the array from which a range is to be copied 352 | * @param from the initial index of the range to be copied, inclusive 353 | * @param to the final index of the range to be copied, exclusive. 354 | * (This index may lie outside the array.) 355 | * @return a new array containing the specified range from the original array, 356 | * truncated or padded with zeros to obtain the required length 357 | * @throws ArrayIndexOutOfBoundsException if from < 0 358 | * or from > original.length() 359 | * @throws IllegalArgumentException if from > to 360 | * @throws NullPointerException if original is null 361 | * @since 1.6 362 | */ 363 | public static byte[] copyOfRange(byte[] original, int from, int to) { 364 | int newLength = to - from; 365 | if (newLength < 0) 366 | throw new IllegalArgumentException(from + " > " + to); 367 | byte[] copy = new byte[newLength]; 368 | System.arraycopy(original, from, copy, 0, 369 | Math.min(original.length - from, newLength)); 370 | return copy; 371 | } 372 | 373 | private static final ObjectCache> clazzCache = (new SimpleObjectCacheFactoryImpl(256)).createObjectCache(); 374 | 375 | @SuppressWarnings("unchecked") 376 | public static final Object createObject(String className) throws Exception { 377 | Asserts.stringNotEmpty("className", className); 378 | Class clazz = clazzCache.get(className); 379 | if (clazz == null) { 380 | ClassLoader cl = retrieveClassLoader(); 381 | clazz = (Class) cl.loadClass(className); 382 | } 383 | 384 | Object obj = clazz.newInstance(); 385 | if (clazzCache.get(className) == null) { 386 | synchronized (Utils.class) { 387 | if (clazzCache.get(className) == null) { 388 | clazzCache.put(className, clazz); 389 | } 390 | } 391 | } 392 | 393 | return obj; 394 | } 395 | 396 | public static final int combine4Bytes2Int(int byte1, int byte2, int byte3, int byte4) { 397 | return (((byte1 << 24) & 0xFF000000) | 398 | ((byte2 << 16) & 0x00FF0000) | 399 | ((byte3 << 8) & 0x0000FF00) | 400 | (byte4 & 0x000000FF)); 401 | } 402 | 403 | } 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | -------------------------------------------------------------------------------- /code/src/main/resources/for-location.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garychenc/file-map/d2ad0a7d44099d03ce98e3eb7cee0a26827a41db/code/src/main/resources/for-location.txt -------------------------------------------------------------------------------- /code/src/test/resources/for-location.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garychenc/file-map/d2ad0a7d44099d03ce98e3eb7cee0a26827a41db/code/src/test/resources/for-location.txt -------------------------------------------------------------------------------- /doc/img/Overview-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garychenc/file-map/d2ad0a7d44099d03ce98e3eb7cee0a26827a41db/doc/img/Overview-1.png -------------------------------------------------------------------------------- /doc/img/Overview-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garychenc/file-map/d2ad0a7d44099d03ce98e3eb7cee0a26827a41db/doc/img/Overview-2.png -------------------------------------------------------------------------------- /doc/img/Sys-Arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garychenc/file-map/d2ad0a7d44099d03ce98e3eb7cee0a26827a41db/doc/img/Sys-Arch.png --------------------------------------------------------------------------------