├── LICENSE-2.0.txt ├── README.md ├── examples └── java │ └── net │ └── smacke │ └── jaydio │ └── examples │ └── JaydioCp.java ├── pom.xml └── src ├── log4j.properties ├── main └── java │ └── net │ └── smacke │ └── jaydio │ ├── DirectIoLib.java │ ├── DirectRandomAccessFile.java │ ├── OpenFlags.java │ ├── align │ ├── ByteChannelAligner.java │ ├── DirectIoByteChannelAligner.java │ ├── MockByteChannelAligner.java │ └── MockDirectIoLib.java │ ├── buffer │ ├── AbstractBuffer.java │ ├── AlignedDirectByteBuffer.java │ ├── Buffer.java │ ├── JavaHeapByteBuffer.java │ └── JaydioByteBuffer.java │ └── channel │ ├── BufferedChannel.java │ ├── DirectIoByteChannel.java │ └── MockByteChannel.java └── test └── java └── net └── smacke └── jaydio ├── align ├── TestAlignedIO.java └── TestDirectIO.java ├── buffer ├── AbstractBufferTester.java ├── TestAlignedDirectByteBuffer.java └── TestJavaHeapByteBuffer.java └── channel ├── BufferedChannelAbstractTester.java ├── TestDirectIoByteChannel.java └── TestMockByteChannel.java /LICENSE-2.0.txt: -------------------------------------------------------------------------------- 1 | ==== 2 | Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | ==== 16 | 17 | 18 | Apache License 19 | Version 2.0, January 2004 20 | http://www.apache.org/licenses/ 21 | 22 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 23 | 24 | 1. Definitions. 25 | 26 | "License" shall mean the terms and conditions for use, reproduction, 27 | and distribution as defined by Sections 1 through 9 of this document. 28 | 29 | "Licensor" shall mean the copyright owner or entity authorized by 30 | the copyright owner that is granting the License. 31 | 32 | "Legal Entity" shall mean the union of the acting entity and all 33 | other entities that control, are controlled by, or are under common 34 | control with that entity. For the purposes of this definition, 35 | "control" means (i) the power, direct or indirect, to cause the 36 | direction or management of such entity, whether by contract or 37 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 38 | outstanding shares, or (iii) beneficial ownership of such entity. 39 | 40 | "You" (or "Your") shall mean an individual or Legal Entity 41 | exercising permissions granted by this License. 42 | 43 | "Source" form shall mean the preferred form for making modifications, 44 | including but not limited to software source code, documentation 45 | source, and configuration files. 46 | 47 | "Object" form shall mean any form resulting from mechanical 48 | transformation or translation of a Source form, including but 49 | not limited to compiled object code, generated documentation, 50 | and conversions to other media types. 51 | 52 | "Work" shall mean the work of authorship, whether in Source or 53 | Object form, made available under the License, as indicated by a 54 | copyright notice that is included in or attached to the work 55 | (an example is provided in the Appendix below). 56 | 57 | "Derivative Works" shall mean any work, whether in Source or Object 58 | form, that is based on (or derived from) the Work and for which the 59 | editorial revisions, annotations, elaborations, or other modifications 60 | represent, as a whole, an original work of authorship. For the purposes 61 | of this License, Derivative Works shall not include works that remain 62 | separable from, or merely link (or bind by name) to the interfaces of, 63 | the Work and Derivative Works thereof. 64 | 65 | "Contribution" shall mean any work of authorship, including 66 | the original version of the Work and any modifications or additions 67 | to that Work or Derivative Works thereof, that is intentionally 68 | submitted to Licensor for inclusion in the Work by the copyright owner 69 | or by an individual or Legal Entity authorized to submit on behalf of 70 | the copyright owner. For the purposes of this definition, "submitted" 71 | means any form of electronic, verbal, or written communication sent 72 | to the Licensor or its representatives, including but not limited to 73 | communication on electronic mailing lists, source code control systems, 74 | and issue tracking systems that are managed by, or on behalf of, the 75 | Licensor for the purpose of discussing and improving the Work, but 76 | excluding communication that is conspicuously marked or otherwise 77 | designated in writing by the copyright owner as "Not a Contribution." 78 | 79 | "Contributor" shall mean Licensor and any individual or Legal Entity 80 | on behalf of whom a Contribution has been received by Licensor and 81 | subsequently incorporated within the Work. 82 | 83 | 2. Grant of Copyright License. Subject to the terms and conditions of 84 | this License, each Contributor hereby grants to You a perpetual, 85 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 86 | copyright license to reproduce, prepare Derivative Works of, 87 | publicly display, publicly perform, sublicense, and distribute the 88 | Work and such Derivative Works in Source or Object form. 89 | 90 | 3. Grant of Patent License. Subject to the terms and conditions of 91 | this License, each Contributor hereby grants to You a perpetual, 92 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 93 | (except as stated in this section) patent license to make, have made, 94 | use, offer to sell, sell, import, and otherwise transfer the Work, 95 | where such license applies only to those patent claims licensable 96 | by such Contributor that are necessarily infringed by their 97 | Contribution(s) alone or by combination of their Contribution(s) 98 | with the Work to which such Contribution(s) was submitted. If You 99 | institute patent litigation against any entity (including a 100 | cross-claim or counterclaim in a lawsuit) alleging that the Work 101 | or a Contribution incorporated within the Work constitutes direct 102 | or contributory patent infringement, then any patent licenses 103 | granted to You under this License for that Work shall terminate 104 | as of the date such litigation is filed. 105 | 106 | 4. Redistribution. You may reproduce and distribute copies of the 107 | Work or Derivative Works thereof in any medium, with or without 108 | modifications, and in Source or Object form, provided that You 109 | meet the following conditions: 110 | 111 | (a) You must give any other recipients of the Work or 112 | Derivative Works a copy of this License; and 113 | 114 | (b) You must cause any modified files to carry prominent notices 115 | stating that You changed the files; and 116 | 117 | (c) You must retain, in the Source form of any Derivative Works 118 | that You distribute, all copyright, patent, trademark, and 119 | attribution notices from the Source form of the Work, 120 | excluding those notices that do not pertain to any part of 121 | the Derivative Works; and 122 | 123 | (d) If the Work includes a "NOTICE" text file as part of its 124 | distribution, then any Derivative Works that You distribute must 125 | include a readable copy of the attribution notices contained 126 | within such NOTICE file, excluding those notices that do not 127 | pertain to any part of the Derivative Works, in at least one 128 | of the following places: within a NOTICE text file distributed 129 | as part of the Derivative Works; within the Source form or 130 | documentation, if provided along with the Derivative Works; or, 131 | within a display generated by the Derivative Works, if and 132 | wherever such third-party notices normally appear. The contents 133 | of the NOTICE file are for informational purposes only and 134 | do not modify the License. You may add Your own attribution 135 | notices within Derivative Works that You distribute, alongside 136 | or as an addendum to the NOTICE text from the Work, provided 137 | that such additional attribution notices cannot be construed 138 | as modifying the License. 139 | 140 | You may add Your own copyright statement to Your modifications and 141 | may provide additional or different license terms and conditions 142 | for use, reproduction, or distribution of Your modifications, or 143 | for any such Derivative Works as a whole, provided Your use, 144 | reproduction, and distribution of the Work otherwise complies with 145 | the conditions stated in this License. 146 | 147 | 5. Submission of Contributions. Unless You explicitly state otherwise, 148 | any Contribution intentionally submitted for inclusion in the Work 149 | by You to the Licensor shall be under the terms and conditions of 150 | this License, without any additional terms or conditions. 151 | Notwithstanding the above, nothing herein shall supersede or modify 152 | the terms of any separate license agreement you may have executed 153 | with Licensor regarding such Contributions. 154 | 155 | 6. Trademarks. This License does not grant permission to use the trade 156 | names, trademarks, service marks, or product names of the Licensor, 157 | except as required for reasonable and customary use in describing the 158 | origin of the Work and reproducing the content of the NOTICE file. 159 | 160 | 7. Disclaimer of Warranty. Unless required by applicable law or 161 | agreed to in writing, Licensor provides the Work (and each 162 | Contributor provides its Contributions) on an "AS IS" BASIS, 163 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 164 | implied, including, without limitation, any warranties or conditions 165 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 166 | PARTICULAR PURPOSE. You are solely responsible for determining the 167 | appropriateness of using or redistributing the Work and assume any 168 | risks associated with Your exercise of permissions under this License. 169 | 170 | 8. Limitation of Liability. In no event and under no legal theory, 171 | whether in tort (including negligence), contract, or otherwise, 172 | unless required by applicable law (such as deliberate and grossly 173 | negligent acts) or agreed to in writing, shall any Contributor be 174 | liable to You for damages, including any direct, indirect, special, 175 | incidental, or consequential damages of any character arising as a 176 | result of this License or out of the use or inability to use the 177 | Work (including but not limited to damages for loss of goodwill, 178 | work stoppage, computer failure or malfunction, or any and all 179 | other commercial damages or losses), even if such Contributor 180 | has been advised of the possibility of such damages. 181 | 182 | 9. Accepting Warranty or Additional Liability. While redistributing 183 | the Work or Derivative Works thereof, You may choose to offer, 184 | and charge a fee for, acceptance of support, warranty, indemnity, 185 | or other liability obligations and/or rights consistent with this 186 | License. However, in accepting such obligations, You may act only 187 | on Your own behalf and on Your sole responsibility, not on behalf 188 | of any other Contributor, and only if You agree to indemnify, 189 | defend, and hold each Contributor harmless for any liability 190 | incurred by, or claims asserted against, such Contributor by reason 191 | of your accepting any such warranty or additional liability. 192 | 193 | END OF TERMS AND CONDITIONS 194 | 195 | APPENDIX: How to apply the Apache License to your work. 196 | 197 | To apply the Apache License to your work, attach the following 198 | boilerplate notice, with the fields enclosed by brackets "[]" 199 | replaced with your own identifying information. (Don't include 200 | the brackets!) The text should be enclosed in the appropriate 201 | comment syntax for the file format. We also recommend that a 202 | file or class name and description of purpose be included on the 203 | same "printed page" as the copyright notice for easier 204 | identification within third-party archives. 205 | 206 | Copyright [yyyy] [name of copyright owner] 207 | 208 | Licensed under the Apache License, Version 2.0 (the "License"); 209 | you may not use this file except in compliance with the License. 210 | You may obtain a copy of the License at 211 | 212 | http://www.apache.org/licenses/LICENSE-2.0 213 | 214 | Unless required by applicable law or agreed to in writing, software 215 | distributed under the License is distributed on an "AS IS" BASIS, 216 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 217 | See the License for the specific language governing permissions and 218 | limitations under the License. 219 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Java Direct I/O (Jaydio) 2 | ======================== 3 | 4 | Jaydio is a Java library for giving the programmer finer control over file I/O, 5 | in part by bypassing the OS buffer cache. **For now, only Linux is supported**. 6 | Jaydio serves as as a starting point for asynchronous / non-blocking direct 7 | I/O, custom file page caching layers, and even advanced cache replacement 8 | policies like [ARC](http://dbs.uni-leipzig.de/file/ARC.pdf) or 9 | [2Q](http://www.inf.fu-berlin.de/lehre/WS06/DBS-Tech/Reader/2QBufferManagement.pdf). 10 | Currently, it is useful for preventing the operating system from evicting 11 | memory pages which need to stay "warm" in favor of non-critical file pages. 12 | 13 | 14 | 15 | Adding Jaydio to Your Project 16 | ----------------------------- 17 | 18 | [jaydio-0.1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/net/smacke/jaydio/0.1/jaydio-0.1.jar) 19 | 20 | Jaydio has dependencies on [JNA](https://github.com/twall/jna) and [SLF4J](http://www.slf4j.org/). 21 | 22 | If you use Maven, you can add the following to your `pom.xml`: 23 | 24 | ```xml 25 | 26 | ... 27 | 28 | net.smacke 29 | jaydio 30 | 0.1 31 | 32 | ... 33 | 34 | ``` 35 | 36 | Alternatively, if you don't want to deal with Maven or with tracking down 37 | dependencies manually, the following are also available: 38 | 39 | [jaydio-0.1-jar-with-dependencies.jar](http://smacke.net/jaydio/jaydio-0.1-jar-with-dependencies.jar) 40 | 41 | [jaydio-0.1-jar-with-dependencies.jar.asc](http://smacke.net/jaydio/jaydio-0.1-jar-with-dependencies.jar.asc) 42 | (if you want to verify the jar against my gpg public key) 43 | 44 | The API reference for Jaydio is located in the 45 | [javadoc](http://smacke.net/jaydio/javadoc/index.html). 46 | 47 | How it works 48 | ------------ 49 | 50 | Jaydio works by passing the `O_DIRECT` flag to the Linux `open()` system call. 51 | Unfortunately, talking directly to disk imposes some nasty rules about how many 52 | bytes can be written at a time, and where they can be written. In particular, 53 | all writes must be in multiples of the file system block size, and they must 54 | originate from chunks of memory aligned to the memory page size, and they must 55 | also occur at file system block boundaries. see `man 2 open` (search for 56 | `O_DIRECT`) and `man 3 posix_memalign` for more details. 57 | 58 | Fortunately, with Jaydio you don't have to worry about any of that. All of the 59 | nasty alignment rule parameters are determined on-the-fly, so that native libc 60 | file reads and writes may be used to bypass the OS cache. Jaydio uses 61 | [JNA](https://github.com/twall/jna/) to accomplish all this parameter querying 62 | and other native magic. 63 | 64 | Example 65 | ------- 66 | 67 | ```java 68 | int bufferSize = 1<<23; // Use 8 MiB buffers 69 | byte[] buf = new byte[bufferSize]; 70 | 71 | DirectRandomAccessFile fin = 72 | new DirectRandomAccessFile(new File("hello.txt"), "r", bufferSize); 73 | 74 | DirectRandomAccessFile fout = 75 | new DirectRandomAccessFile(new File("world.txt"), "rw", bufferSize); 76 | 77 | while (fin.getFilePointer() < fin.length()) { 78 | int remaining = (int)Math.min(bufferSize, fin.length()-fin.getFilePointer()); 79 | fin.read(buf,0,remaining); 80 | fout.write(buf,0,remaining); 81 | } 82 | 83 | fin.close(); 84 | fout.close(); 85 | ``` 86 | 87 | FAQ 88 | === 89 | 90 | #### Does it work? 91 | 92 | Here is a plot of memory use on my system during a copy of a 1.4 GB file, under 93 | different scenarios. The first scenario (red) corresponds to having plenty of 94 | memory free (about 2.6 GB) **at the start of the copy**, so that, if need be, 95 | we could basically fit two copies of file in memory. The second scenario 96 | (blue), my system has limited memory available (about 900 MB) when copying 97 | begins. The third scenario (green), we use Jaydio for the file copy (yay!), 98 | with buffers totaling about 36 MiB. For each scenario, completion of the copy 99 | is indicated by a vertical dotted line. 100 | 101 | ![File copy comparison plot](https://raw.github.com/smacke/jaydio/gh-pages/jaydio-cp-plot.png) 102 | 103 | As you can see, Jaydio successfully bypasses the OS buffer cache, and uses no 104 | memory other than the small buffers allocated to it. Interestingly enough, for 105 | scenario 2 (blue, RAM limited), we're only taking marginally less time than 106 | direct I/O for the copy! This is likely because we ran out of RAM early on, and 107 | so are effecively doing direct I/O at the point where the blue curve flatlines, 108 | as we need to free up space in the cache whenever we do a write. 109 | 110 | Jaydio takes about 41 seconds to do the copy, RAM-limited `cp` takes about 40.5 111 | seconds, and RAM-abundant `cp` takes about 26.8 seconds on my system, for a 1.4 112 | GB file. 113 | 114 | #### Why bypass file system cache? Isn't it there for a reason? 115 | 116 | Yes, and in 99% of cases you will want to use it. As mentioned earlier, Jaydio 117 | serves as a starting point for implementing more advanced cache replacement 118 | policies than LRU, but there are also times when you want to prevent the 119 | operating system from evicting certain memory pages in favor of caching files. 120 | 121 | 122 | #### Can't I just use something like `DONTNEED` or `NOREUSE` with `posix_fadvise` or `madvise` if I'm worried about that page cache stomping over useful memory pages? 123 | 124 | As far as I know, `NOREUSE` is a no-op. 125 | 126 | You could use `DONTNEED`, and you would likely see better performance (in terms 127 | of raw speed), too. Unfortunately, this sort of strategy does not discriminate 128 | between cached pages that you want to save and those that you want to write 129 | back to disk, if, say, the same file has been opened by multiple threads, of 130 | differing priorities regarding memory accesses. 131 | [Here](http://blog.mikemccandless.com/2010/06/lucene-and-fadvisemadvise.html) 132 | is one example of a situation where this sort of strategy will not work. 133 | 134 | 135 | #### I'm on Linux, but when I try to create a Jaydio `DirectRandomAccessFile` I get "Error opening file, got Invalid argument". What gives? 136 | 137 | This is the very uninformative libc error that you get when something goes 138 | wrong when opening a file. Chances are that your file system does not support 139 | opening files with the `O_DIRECT` flag. For example, if you encrypt your home 140 | directory using a file system like ecryptfs, you can't use direct I/O there. 141 | 142 | 143 | Contributing 144 | ============ 145 | 146 | Any contribution, whether it be a bug report or new core functionality, is 147 | welcome. Please feel free to fork and send pull requests! That being said, 148 | contributions which include unit tests are preferred. 149 | 150 | Once you have cloned the source, one of the easier ways to get set up is with 151 | eclipse, especially if you have the [m2eclipse](https://www.eclipse.org/m2e/) 152 | plugin. Just go to "File -> New -> Project -> Maven Project" and point the root 153 | directory to wherever you cloned Jaydio. 154 | 155 | If you don't have `m2eclipse`, you can still set things up easily if you run 156 | `mvn eclipse:eclipse` wherever you cloned Jaydio to set up `.project` and 157 | `.classpath` files for eclipse. From eclipse, then do "File -> New -> Java 158 | Project" and point the root project directory to the Jaydio directory. 159 | 160 | Lastly, if you find some use for Jaydio, one excellent way to contribute is to 161 | shoot me an email describing what you're doing -- I'd love to hear about it! 162 | 163 | License 164 | ======= 165 | 166 | This library is released under the Apache Software License, version 2.0. 167 | -------------------------------------------------------------------------------- /examples/java/net/smacke/jaydio/examples/JaydioCp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.examples; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | 21 | import net.smacke.jaydio.DirectRandomAccessFile; 22 | 23 | public class JaydioCp { 24 | 25 | public static void main(String[] args) throws IOException { 26 | 27 | int bufferSize = 28 | // use 8 MiB buffers by default 29 | args.length >= 3 ? Integer.parseInt(args[3]) : (1<<23); 30 | 31 | byte[] buf = new byte[bufferSize]; 32 | 33 | DirectRandomAccessFile fin = 34 | new DirectRandomAccessFile(new File(args[0]), "r", bufferSize); 35 | 36 | DirectRandomAccessFile fout = 37 | new DirectRandomAccessFile(new File(args[1]), "rw", bufferSize); 38 | 39 | while (fin.getFilePointer() < fin.length()) { 40 | int remaining = (int)Math.min(bufferSize, fin.length()-fin.getFilePointer()); 41 | fin.read(buf,0,remaining); 42 | fout.write(buf,0,remaining); 43 | } 44 | 45 | fin.close(); 46 | fout.close(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | net.smacke 5 | jaydio 6 | 0.2-SNAPSHOT 7 | jar 8 | 2014 9 | 10 | Jaydio 11 | A Java library for tighter control of file I/O. 12 | https://github.com/smacke/jaydio 13 | 14 | 15 | org.sonatype.oss 16 | oss-parent 17 | 7 18 | 19 | 20 | 21 | 22 | The Apache Software License, Version 2.0 23 | http://www.apache.org/licenses/LICENSE-2.0.txt 24 | repo 25 | A business-friendly OSS license 26 | 27 | 28 | 29 | 30 | 31 | smacke 32 | Stephen Macke 33 | smacke@cs.stanford.edu 34 | http://smacke.net 35 | 36 | Owner 37 | 38 | -8 39 | 40 | http://www.gravatar.com/avatar/57426781225d355bf6305dc79663c1b3.png 41 | 42 | 43 | 44 | 45 | 46 | scm:git:https://github.com/smacke/jaydio 47 | scm:git:git@github.com:smacke/jaydio.git 48 | https://github.com/smacke/jaydio 49 | 50 | 51 | 52 | GitHub 53 | https://github.com/smacke/jaydio/issues 54 | 55 | 56 | 57 | UTF-8 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-compiler-plugin 66 | 2.1 67 | 68 | 1.6 69 | 1.6 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | junit 78 | junit 79 | 4.13.1 80 | test 81 | 82 | 83 | net.java.dev.jna 84 | jna 85 | 4.0.0 86 | 87 | 88 | org.slf4j 89 | slf4j-api 90 | 1.7.32 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | ### direct log messages to stdout ### 18 |
log4j.appender.stdout=org.apache.log4j.ConsoleAppender 19 | log4j.appender.stdout.Target=System.out 20 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 21 | log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n 22 | log4j.rootLogger=debug, stdout 23 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/DirectIoLib.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio; 17 | 18 | import java.io.EOFException; 19 | import java.io.IOException; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import net.smacke.jaydio.buffer.AlignedDirectByteBuffer; 24 | 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import com.sun.jna.Native; 29 | import com.sun.jna.NativeLong; 30 | import com.sun.jna.Platform; 31 | import com.sun.jna.Pointer; 32 | import com.sun.jna.ptr.PointerByReference; 33 | 34 | /** 35 | * Class containing native hooks and utility methods for performing direct I/O, using 36 | * the Linux O_DIRECT flag.

37 | * 38 | *

This class is initialized at class load time, by registering JNA hooks into native methods. 39 | * It also calculates Linux kernel version-dependent alignment amount (in bytes) for use with the O_DIRECT flag, 40 | * when given a string for a file or directory.

41 | * 42 | * @author smacke 43 | * 44 | */ 45 | public class DirectIoLib { 46 | private static final Logger logger = LoggerFactory.getLogger(DirectIoLib.class); 47 | private static boolean binit; 48 | 49 | static { 50 | binit = false; 51 | try { 52 | if (!Platform.isLinux()) { // TODO (smacke): test on other *nix variants 53 | logger.warn("Not running Linux, jaydio support disabled"); 54 | } else { // now check to see if we have O_DIRECT... 55 | 56 | final int linuxVersion = 0; 57 | final int majorRev = 1; 58 | final int minorRev = 2; 59 | 60 | List versionNumbers = new ArrayList(); 61 | for (String v : System.getProperty("os.version").split("\\.|-")) { 62 | if (v.matches("\\d")) { 63 | versionNumbers.add(Integer.parseInt(v)); 64 | } 65 | } 66 | 67 | /* From "man 2 open": 68 | * 69 | * O_DIRECT support was added under Linux in kernel version 2.4.10. Older Linux kernels simply ignore this flag. Some file systems may not implement 70 | * the flag and open() will fail with EINVAL if it is used. 71 | */ 72 | 73 | // test to see whether kernel version >= 2.4.10 74 | if (versionNumbers.get(linuxVersion) > 2) { 75 | binit = true; 76 | } else if (versionNumbers.get(linuxVersion) == 2) { 77 | if (versionNumbers.get(majorRev) > 4) { 78 | binit = true; 79 | } else if (versionNumbers.get(majorRev) == 4 && versionNumbers.get(minorRev) >= 10) { 80 | binit = true; 81 | } 82 | } 83 | 84 | if (binit) { 85 | Native.register(Platform.C_LIBRARY_NAME); // get access to open(), pread(), etc 86 | } else { 87 | logger.warn(String.format("O_DIRECT not supported on your version of Linux: %d.%d.%d", linuxVersion, majorRev, minorRev)); 88 | } 89 | } 90 | } catch (Throwable e) { 91 | logger.warn("Unable to register libc at class load time: " + e.getMessage(), e); 92 | } 93 | } 94 | 95 | private int fsBlockSize; 96 | private long fsBlockNotMask; 97 | 98 | // protected for tests 99 | protected DirectIoLib(int fsBlockSize) { 100 | this.fsBlockSize = fsBlockSize; 101 | this.fsBlockNotMask = ~((long)fsBlockSize - 1); 102 | } 103 | 104 | 105 | /** 106 | * Static method to register JNA hooks for doing direct I/O

107 | * 108 | * @param workingDir 109 | * A directory within the mounted file system on which we'll be working 110 | * Should preferably BE the directory in which we'll be working. 111 | */ 112 | public static DirectIoLib getLibForPath(String workingDir) { 113 | int fsBlockSize = initilizeSoftBlockSize(workingDir); 114 | if (fsBlockSize == -1) { 115 | logger.warn("O_DIRECT support non available on your version of Linux (" + System.getProperty("os.version") + "), " + 116 | "please upgrade your kernel in order to use jaydio."); 117 | return null; 118 | } 119 | return new DirectIoLib(fsBlockSize); 120 | } 121 | 122 | /** 123 | * Finds a block size for use with O_DIRECT. Choose it in the most paranoid 124 | * way possible to maximize probability that things work. 125 | * 126 | * @param fileOrDir 127 | * A file or directory within which O_DIRECT access will be performed. 128 | */ 129 | private static int initilizeSoftBlockSize(String fileOrDir) { 130 | 131 | int fsBlockSize = -1; 132 | 133 | if (binit) { 134 | // get file system block size for use with workingDir 135 | // see "man 3 posix_memalign" for why we do this 136 | final int _PC_REC_XFER_ALIGN = 0x11; 137 | 138 | fsBlockSize = pathconf(fileOrDir, _PC_REC_XFER_ALIGN); 139 | /* conservative for version >= 2.6 140 | * "man 2 open": 141 | * 142 | * Under Linux 2.6, alignment 143 | * to 512-byte boundaries suffices. 144 | */ 145 | 146 | // Since O_DIRECT requires pages to be memory aligned with the file system block size, 147 | // we will do this too in case the page size and the block size are different for 148 | // whatever reason. By taking the least common multiple, everything should be happy: 149 | int pageSize = getpagesize(); 150 | fsBlockSize = lcm(fsBlockSize, pageSize); 151 | 152 | // just being completely paranoid: 153 | // (512 is the rule for 2.6+ kernels as mentioned before) 154 | fsBlockSize = lcm(fsBlockSize, 512); 155 | 156 | // lastly, a sanity check 157 | if (fsBlockSize <= 0 || ((fsBlockSize & (fsBlockSize-1)) != 0)) { 158 | logger.warn("file system block size should be a power of two, was found to be " + fsBlockSize); 159 | logger.warn("Disabling O_DIRECT support"); 160 | return -1; 161 | } 162 | } 163 | 164 | return fsBlockSize; 165 | } 166 | 167 | 168 | // -- Java interfaces to native methods 169 | 170 | /** 171 | * Interface into native pread function. Always reads an entire buffer, 172 | * unlike {@link #pwrite(int, AlignedDirectByteBuffer, long) pwrite()} which uses buffer state 173 | * to determine how much of buffer to write.

174 | * 175 | * @param fd 176 | * A file discriptor to pass to native pread 177 | * 178 | * @param buf 179 | * The buffer into which to record the file read 180 | * 181 | * @param offset 182 | * The file offset at which to read 183 | * 184 | * @return The number of bytes successfully read from the file 185 | * 186 | * @throws IOException 187 | */ 188 | public int pread(int fd, AlignedDirectByteBuffer buf, long offset) throws IOException { 189 | buf.clear(); // so that we read an entire buffer 190 | int n = pread(fd, buf.pointer(), new NativeLong(buf.capacity()), new NativeLong(offset)).intValue(); 191 | if (n==0) throw new EOFException("Tried to read past EOF at offset " + offset + " into ByteBuffer " + buf); 192 | if (n < 0) { 193 | throw new IOException("error reading file at offset " + offset + ": " + getLastError()); 194 | } 195 | return n; 196 | } 197 | 198 | /** 199 | * Interface into native pwrite function. Writes bytes corresponding to the nearest file 200 | * system block boundaries between buf.position() and buf.limit().

201 | * 202 | * @param fd 203 | * A file descriptor to pass to native pwrite 204 | * 205 | * @param buf 206 | * The buffer from which to write 207 | * 208 | * @param offset 209 | * The file offset at which to write 210 | * 211 | * @return The number of bytes successfully written to the file 212 | * 213 | * @throws IOException 214 | */ 215 | public int pwrite(int fd, AlignedDirectByteBuffer buf, long offset) throws IOException { 216 | 217 | // must always write to end of current block 218 | // To handle writes past the logical file size, 219 | // we will later truncate. 220 | final int start = buf.position(); 221 | assert start == blockStart(start); 222 | final int toWrite = blockEnd(buf.limit()) - start; 223 | 224 | int n = pwrite(fd, buf.pointer().share(start), new NativeLong(toWrite), new NativeLong(offset)).intValue(); 225 | if (n < 0) { 226 | throw new IOException("error writing file at offset " + offset + ": " + getLastError()); 227 | } 228 | return n; 229 | } 230 | 231 | /** 232 | * Use the open Linux system call and pass in the O_DIRECT flag. 233 | * Currently the only other flags passed in are O_RDONLY if readOnly 234 | * is true, and (if not) O_RDWR and O_CREAT. 235 | * 236 | * @param pathname 237 | * The path to the file to open. If file does not exist and we are opening 238 | * with readOnly, this will throw an error. Otherwise, if it does 239 | * not exist but we have readOnly set to false, create the file. 240 | * 241 | * @param readOnly 242 | * Whether to pass in O_RDONLY 243 | * 244 | * @return An integer file descriptor for the opened file 245 | * 246 | * @throws IOException 247 | */ 248 | public int oDirectOpen(String pathname, boolean readOnly) throws IOException { 249 | int flags = OpenFlags.O_DIRECT; 250 | if (readOnly) { 251 | flags |= OpenFlags.O_RDONLY; 252 | } else { 253 | flags |= OpenFlags.O_RDWR | OpenFlags.O_CREAT; 254 | } 255 | int fd = open(pathname, flags, 00644); 256 | if (fd < 0) { 257 | throw new IOException("Error opening " + pathname + ", got " + getLastError()); 258 | } 259 | return fd; 260 | } 261 | 262 | /** 263 | * Hooks into errno using Native.getLastError(), and parses it with native strerror function. 264 | * 265 | * @return An error message corresponding to the last errno 266 | */ 267 | public static String getLastError() { 268 | return strerror(Native.getLastError()); 269 | } 270 | 271 | 272 | // -- alignment logic utility methods 273 | 274 | /** 275 | * @return The soft block size for use with transfer multiples 276 | * and memory alignment multiples 277 | */ 278 | public int blockSize() { 279 | return fsBlockSize; 280 | } 281 | 282 | /** 283 | * Returns the default buffer size for file channels doing O_DIRECT 284 | * I/O. By default this is equal to the block size. 285 | * 286 | * @return The default buffer size 287 | */ 288 | public int defaultBufferSize() { 289 | return fsBlockSize; 290 | } 291 | 292 | /** 293 | * Given value, find the largest number less than or equal 294 | * to value which is a multiple of the fs block size. 295 | * 296 | * @param value 297 | * @return The largest number less than or equal to value 298 | * which is a multiple of the soft block size 299 | */ 300 | public long blockStart(long value) { 301 | return value & fsBlockNotMask; 302 | } 303 | 304 | 305 | /** 306 | * @see #blockStart(long) 307 | */ 308 | public int blockStart(int value) { 309 | return (int) (value & fsBlockNotMask); 310 | } 311 | 312 | 313 | /** 314 | * Given value, find the smallest number greater than or equal 315 | * to value which is a multiple of the fs block size. 316 | * 317 | * @param value 318 | * @return The smallest number greater than or equal to value 319 | * which is a multiple of the soft block size 320 | */ 321 | public long blockEnd(long value) { 322 | return (value + fsBlockSize- 1) & fsBlockNotMask; 323 | } 324 | 325 | 326 | 327 | /** 328 | * @see #blockEnd(long) 329 | */ 330 | public int blockEnd(int value) { 331 | return (int) ((value + fsBlockSize - 1) & fsBlockNotMask); 332 | } 333 | 334 | 335 | /** 336 | * Static variant of {@link #blockEnd(int)}. 337 | * @param blockSize 338 | * @param position 339 | * @return The smallest number greater than or equal to position 340 | * which is a multiple of the blockSize 341 | */ 342 | public static long blockEnd(int blockSize, long position) { 343 | long ceil = (position + blockSize - 1)/blockSize; 344 | return ceil*blockSize; 345 | } 346 | 347 | 348 | /** 349 | * @param x 350 | * @param y 351 | * @return The least common multiple of x and y 352 | */ 353 | // Euclid's algo for gcd is more general than we need 354 | // since we only have powers of 2, but w/e 355 | public static int lcm(long x, long y) { 356 | long g = x; // will hold gcd 357 | long yc = y; 358 | 359 | // get the gcd first 360 | while (yc != 0) { 361 | long t = g; 362 | g = yc; 363 | yc = t % yc; 364 | } 365 | 366 | return (int)(x*y/g); 367 | } 368 | 369 | 370 | /** 371 | * Given a pointer-to-pointer memptr, sets the dereferenced value to point to the start 372 | * of an allocated block of size bytes, where the starting address is a multiple of 373 | * alignment. It is guaranteed that the block may be freed by calling @{link {@link #free(Pointer)} 374 | * on the starting address. See "man 3 posix_memalign". 375 | * 376 | * @param memptr The pointer-to-pointer which will point to the address of the allocated aligned block 377 | * 378 | * @param alignment The alignment multiple of the starting address of the allocated block 379 | * 380 | * @param size The number of bytes to allocate 381 | * 382 | * @return 0 on success, one of the C error codes on failure. 383 | */ 384 | public static native int posix_memalign(PointerByReference memptr, NativeLong alignment, NativeLong size); 385 | 386 | 387 | /** 388 | * See "man 3 free". 389 | * 390 | * @param ptr The pointer to the hunk of memory which needs freeing 391 | */ 392 | public static native void free(Pointer ptr); 393 | 394 | 395 | /** 396 | * See "man 2 close" 397 | * 398 | * @param fd The file descriptor of the file to close 399 | * 400 | * @return 0 on success, -1 on error 401 | */ 402 | public native int close(int fd); // musn't forget to do this 403 | 404 | 405 | 406 | 407 | // -- more native function hooks -- 408 | 409 | 410 | public static native int ftruncate(int fd, long length); 411 | 412 | private static native NativeLong pwrite(int fd, Pointer buf, NativeLong count, NativeLong offset); 413 | private static native NativeLong pread(int fd, Pointer buf, NativeLong count, NativeLong offset); 414 | private static native int open(String pathname, int flags); 415 | private static native int open(String pathname, int flags, int mode); 416 | private static native int getpagesize(); 417 | private static native int pathconf(String path, int name); 418 | private static native String strerror(int errnum); 419 | 420 | } 421 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/DirectRandomAccessFile.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio; 17 | 18 | import java.io.Closeable; 19 | import java.io.DataInput; 20 | import java.io.DataOutput; 21 | import java.io.EOFException; 22 | import java.io.File; 23 | import java.io.FileNotFoundException; 24 | import java.io.IOException; 25 | import java.io.RandomAccessFile; 26 | 27 | import net.smacke.jaydio.align.DirectIoByteChannelAligner; 28 | 29 | /** 30 | * Class to emulate the behavior of {@link RandomAccessFile}, but using direct I/O. 31 | * @author smacke 32 | * 33 | */ 34 | public class DirectRandomAccessFile implements DataInput, DataOutput, Closeable { 35 | 36 | // "\uFEFF" doesn't seem to work for some reason... hacky workaround 37 | private static final String UTF8_BOM = Character.toString((char)0xEF) + 38 | Character.toString((char)0xBB) + 39 | Character.toString((char)0xBF); 40 | 41 | private DirectIoByteChannelAligner channel; 42 | 43 | /** 44 | * @param name The name of the file to open 45 | * 46 | * @param mode Either "rw" or "r", depending on whether this file is read only 47 | * 48 | * @throws IOException 49 | */ 50 | public DirectRandomAccessFile(String name, String mode) throws IOException { 51 | this(new File(name), mode); 52 | } 53 | 54 | /** 55 | * @param file The file to open 56 | * 57 | * @param mode Either "rw" or "r", depending on whether this file is read only 58 | * 59 | * @throws IOException 60 | */ 61 | public DirectRandomAccessFile(File file, String mode) throws IOException { 62 | this(file, mode, -1); 63 | } 64 | 65 | /** 66 | * @param file The file to open 67 | * 68 | * @param mode Either "rw" or "r", depending on whether this file is read only 69 | * 70 | * @param bufferSize The size of the buffer used to manually buffer I/O 71 | * If -1 the default buffer size is used, which depends on how 72 | * {@link DirectIoLib} is implemented. 73 | * 74 | * @throws IOException 75 | */ 76 | public DirectRandomAccessFile(File file, String mode, int bufferSize) 77 | throws IOException { 78 | 79 | boolean readOnly = false; 80 | if (mode.equals("r")) { 81 | readOnly = true; 82 | } else if (!mode.equals("rw")) { 83 | throw new IllegalArgumentException("only r and rw modes supported"); 84 | } 85 | 86 | if (readOnly && !file.isFile()) { 87 | throw new FileNotFoundException("couldn't find file " + file); 88 | } 89 | 90 | this.channel = bufferSize!=-1 ? 91 | DirectIoByteChannelAligner.open(file, bufferSize, readOnly) : 92 | DirectIoByteChannelAligner.open(file, readOnly); 93 | } 94 | 95 | @Override 96 | public void close() throws IOException { 97 | channel.close(); 98 | } 99 | 100 | @Override 101 | public void write(int v) throws IOException { 102 | channel.write(v); 103 | } 104 | 105 | @Override 106 | public void write(byte[] src) throws IOException { 107 | write(src, 0, src.length); 108 | } 109 | 110 | @Override 111 | public void write(byte[] src, int offset, int length) throws IOException { 112 | channel.writeBytes(src, offset, length); 113 | } 114 | 115 | @Override 116 | public void writeBoolean(boolean b) throws IOException { 117 | write(b ? 1 : 0); 118 | } 119 | 120 | @Override 121 | public void writeByte(int b) throws IOException { 122 | write(b); 123 | } 124 | 125 | @Override 126 | public void writeBytes(String s) throws IOException { 127 | write(s.getBytes()); 128 | } 129 | 130 | @Override 131 | public void writeChar(int c) throws IOException { 132 | write((c >>> 8) & 0xFF); 133 | write((c >>> 0) & 0xFF); 134 | } 135 | 136 | @Override 137 | public void writeChars(String s) throws IOException { 138 | int numChars = s.length(); 139 | int numBytes = 2*numChars; 140 | byte[] sBytes = new byte[numBytes]; 141 | char[] sChars = new char[numChars]; 142 | s.getChars(0, numChars, sChars, 0); 143 | int bPos = 0; 144 | for (int i = 0; i < numChars; i++) { 145 | sBytes[bPos++] = (byte)(sChars[i] >>> 8); 146 | sBytes[bPos++] = (byte)(sChars[i] >>> 0); 147 | } 148 | write(sBytes, 0, numBytes); 149 | } 150 | 151 | @Override 152 | public void writeDouble(double v) throws IOException { 153 | writeLong(Double.doubleToLongBits(v)); 154 | } 155 | 156 | @Override 157 | public void writeFloat(float v) throws IOException { 158 | writeInt(Float.floatToIntBits(v)); 159 | } 160 | 161 | @Override 162 | public void writeInt(int v) throws IOException { 163 | write((v >>> 24) & 0xFF); 164 | write((v >>> 16) & 0xFF); 165 | write((v >>> 8) & 0xFF); 166 | write(v & 0xFF); 167 | } 168 | 169 | @Override 170 | public void writeLong(long v) throws IOException { 171 | write((int)(v >>> 56) & 0xFF); 172 | write((int)(v >>> 48) & 0xFF); 173 | write((int)(v >>> 40) & 0xFF); 174 | write((int)(v >>> 32) & 0xFF); 175 | write((int)(v >>> 24) & 0xFF); 176 | write((int)(v >>> 16) & 0xFF); 177 | write((int)(v >>> 8) & 0xFF); 178 | write((int)v & 0xFF); 179 | } 180 | 181 | @Override 182 | public void writeShort(int v) throws IOException { 183 | write((v >>> 8) & 0xFF); 184 | write((v >>> 0) & 0xFF); 185 | 186 | } 187 | 188 | @Override 189 | public void writeUTF(String s) throws IOException { 190 | // TODO TODO TODO 191 | throw new UnsupportedOperationException("not implemented yet!"); 192 | } 193 | 194 | /** 195 | * Reads one byte, and returns cast as an int. 196 | * @return one byte from the underlying channel 197 | * @throws IOException 198 | */ 199 | public int read() throws IOException { 200 | return channel.read(); 201 | } 202 | 203 | private int readDetectEOF() throws IOException { 204 | int v = read(); 205 | if (v==-1) { 206 | throw new EOFException(); 207 | } 208 | return v; 209 | } 210 | 211 | @Override 212 | public boolean readBoolean() throws IOException { 213 | int b = this.read(); 214 | if (b==-1) { 215 | throw new EOFException(); 216 | } 217 | return b!=0; 218 | } 219 | 220 | @Override 221 | public byte readByte() throws IOException { 222 | int b = this.read(); 223 | if (b==-1) { 224 | throw new EOFException(); 225 | } 226 | return (byte) b; 227 | } 228 | 229 | @Override 230 | public char readChar() throws IOException { 231 | int ch1 = this.read(); 232 | int ch2 = this.read(); 233 | if (ch1==-1 || ch2==-1) { 234 | throw new EOFException(); 235 | } 236 | return (char)((ch1<<8) + ch2); 237 | } 238 | 239 | @Override 240 | public double readDouble() throws IOException { 241 | return Double.longBitsToDouble(readLong()); 242 | } 243 | 244 | @Override 245 | public float readFloat() throws IOException { 246 | return Float.intBitsToFloat(readInt()); 247 | } 248 | 249 | /** 250 | * Bulk read bytes from channel into dst. 251 | * 252 | * @param dst The destination byte array 253 | * 254 | * @param offset The offset within dst to start reading 255 | * 256 | * @param length The number of bytes to read 257 | * 258 | * @throws IOException 259 | */ 260 | public void read(byte[] dst, int offset, int length) throws IOException { 261 | channel.readBytes(dst, offset, length); 262 | } 263 | 264 | /** 265 | * Bulk read bytes from channel into dst. 266 | * 267 | * @param dst The destination byte array 268 | * 269 | * @throws IOException 270 | */ 271 | public void read(byte[] dst) throws IOException { 272 | channel.readBytes(dst, 0, dst.length); 273 | } 274 | 275 | @Override 276 | public void readFully(byte[] src) throws IOException { 277 | readFully(src, 0, src.length); 278 | } 279 | 280 | @Override 281 | public void readFully(byte[] src, int offset, int length) throws IOException { 282 | read(src,offset,length); 283 | } 284 | 285 | @Override 286 | public int readInt() throws IOException { 287 | int b1 = this.readDetectEOF(); 288 | int b2 = this.readDetectEOF(); 289 | int b3 = this.readDetectEOF(); 290 | int b4 = this.readDetectEOF(); 291 | return ((b1<<24) + (b2<<16) + (b3<<8) + b4); 292 | } 293 | 294 | private static String sanitizeUtf8Bom(String s) { 295 | if (s.startsWith(UTF8_BOM)) { 296 | s = s.substring(UTF8_BOM.length()); 297 | } 298 | return s; 299 | } 300 | 301 | @Override 302 | public String readLine() throws IOException { 303 | // boolean sanitizeBOM = getFilePointer()==0; 304 | StringBuilder instr = new StringBuilder(); 305 | int c; 306 | while ((c=read()) != -1) { 307 | if (c=='\n') { 308 | break; 309 | } else if (c=='\r') { 310 | // advance if next char is '\n' 311 | long pos = getFilePointer(); 312 | if (read()!='\n') { 313 | seek(pos); 314 | } 315 | break; 316 | } else { 317 | instr.append((char)c); 318 | } 319 | } 320 | 321 | if (c==-1 && instr.length()==0) { 322 | return null; 323 | } 324 | 325 | String ret = instr.toString(); 326 | // if (sanitizeBOM) { 327 | return sanitizeUtf8Bom(ret); 328 | // } 329 | // return ret; 330 | } 331 | 332 | @Override 333 | public long readLong() throws IOException { 334 | return ((long)(readInt()) << 32) + (readInt() & 0xFFFFFFFFL); 335 | } 336 | 337 | @Override 338 | public short readShort() throws IOException { 339 | int b1 = this.readDetectEOF(); 340 | int b2 = this.readDetectEOF(); 341 | return (short)((b1<<8) + b2); 342 | } 343 | 344 | @Override 345 | public String readUTF() throws IOException { 346 | // TODO Auto-generated method stub 347 | throw new UnsupportedOperationException("not implemented yet!"); 348 | } 349 | 350 | @Override 351 | public int readUnsignedByte() throws IOException { 352 | return readDetectEOF(); 353 | } 354 | 355 | @Override 356 | public int readUnsignedShort() throws IOException { 357 | int b1 = this.readDetectEOF(); 358 | int b2 = this.readDetectEOF(); 359 | return (b1<<8) + b2; 360 | } 361 | 362 | @Override 363 | public int skipBytes(int n) throws IOException { 364 | if (n <= 0) { 365 | return 0; 366 | } 367 | 368 | long pos = getFilePointer(); 369 | long newpos = Math.min(pos + n, length()); 370 | 371 | seek(newpos); 372 | 373 | return (int)(newpos - pos); 374 | } 375 | 376 | /** 377 | * Seeks to position pos within the file. 378 | * @param pos The position to which to seek 379 | * @throws IOException 380 | */ 381 | public void seek(long pos) throws IOException { 382 | channel.position(pos); 383 | } 384 | 385 | /** 386 | * @return The current position in the file 387 | */ 388 | public long getFilePointer() { 389 | return channel.position(); 390 | } 391 | 392 | /** 393 | * @return The current length of the file 394 | */ 395 | public long length() { 396 | return channel.size(); 397 | } 398 | 399 | } 400 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/OpenFlags.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio; 17 | 18 | /** 19 | * Constants for {@link DirectIoLib#oDirectOpen(String, boolean)}.

20 | * 21 | * @author smacke 22 | * 23 | */ 24 | public final class OpenFlags { 25 | public static final int O_RDONLY = 00; 26 | public static final int O_WRONLY = 01; 27 | public static final int O_RDWR = 02; 28 | public static final int O_CREAT = 0100; 29 | public static final int O_TRUNC = 01000; 30 | public static final int O_DIRECT = 040000; 31 | public static final int O_SYNC = 04000000; 32 | 33 | private OpenFlags() {} 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/align/ByteChannelAligner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.align; 17 | 18 | import java.io.EOFException; 19 | import java.io.IOException; 20 | import java.nio.ByteBuffer; 21 | import java.nio.channels.ClosedChannelException; 22 | import java.nio.channels.NonWritableChannelException; 23 | import java.nio.channels.SeekableByteChannel; 24 | import java.util.Arrays; 25 | 26 | import net.smacke.jaydio.DirectIoLib; 27 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 28 | import net.smacke.jaydio.channel.BufferedChannel; 29 | 30 | 31 | /** 32 | * Abstract class which handles reading and writing from/to an underlying byte channel in appropriate 33 | * increments and and at appropriate boundaries of the channel. The motivation for this class is that 34 | * direct I/O in Linux (and in Windows too, I believe) requires all reads and writes to obey these 35 | * strange alignment rules. It may seem like overkill to segment out the alignment logic necessary 36 | * for this into its own abstract class, separate from any actual channel, but this allows for highly 37 | * flexible testing with mock objects, to avoid doing actual heavy-weight I/O when possible.

38 | * 39 | * @author smacke 40 | * 41 | */ 42 | public abstract class ByteChannelAligner implements SeekableByteChannel { 43 | T buffer; 44 | BufferedChannel channel; 45 | 46 | private DirectIoLib lib; 47 | private boolean isOpen; 48 | private long filePos; 49 | private long fileLength; 50 | 51 | private boolean[] dirty; 52 | private boolean globalDirty; 53 | 54 | // TODO (smacke): It may be good to support all the various options that 55 | // Java FileChannel does, e.g. APPEND, TRUNCATE_EXISTING, CREATE_NEW, CREATE, 56 | // DELETE_ON_CLOSE, etc. 57 | 58 | 59 | public ByteChannelAligner(DirectIoLib lib, BufferedChannel channel, T buffer) { 60 | this.lib = lib; 61 | this.buffer = buffer; 62 | this.channel = channel; 63 | this.isOpen = true; 64 | this.fileLength = channel.size(); 65 | dirty = new boolean[buffer.capacity() / lib.blockSize()]; 66 | Arrays.fill(dirty, false); 67 | globalDirty = false; 68 | positionBufferForFlushAndRefill(0); 69 | } 70 | 71 | private void ensureOpen() throws ClosedChannelException { 72 | if (!isOpen) { 73 | throw new ClosedChannelException(); 74 | } 75 | } 76 | 77 | private void ensureWritable() throws NonWritableChannelException { 78 | if (channel.isReadOnly()) { 79 | throw new NonWritableChannelException(); 80 | } 81 | } 82 | 83 | @Override 84 | public void close() throws IOException { 85 | if (isOpen) { 86 | try { 87 | if (!channel.isReadOnly()) { 88 | truncate(size()); 89 | } 90 | } finally { 91 | isOpen = false; 92 | try { 93 | channel.close(); 94 | } finally { 95 | buffer.close(); 96 | } 97 | } 98 | } 99 | } 100 | 101 | @Override 102 | public long position() { 103 | return filePos + buffer.position(); 104 | } 105 | 106 | @Override 107 | public ByteChannelAligner position(long pos) throws IOException { 108 | ensureOpen(); 109 | final long alignedPos = lib.blockStart(pos); 110 | if (filePos != alignedPos) { // if we need to seek() outside the current window 111 | flush(); 112 | if (alignedPos < size()) { 113 | positionBufferForFlushAndRefill(alignedPos); 114 | flushAndRefill(); 115 | } 116 | else { 117 | // now we're past the current channel size 118 | // reads throw EOFException 119 | // writes leave intermediate bytes unspecified 120 | filePos = alignedPos; 121 | } 122 | } 123 | // seek to correct place within buffer window 124 | final int delta = (int) (pos - alignedPos); 125 | buffer.position(delta); 126 | return this; 127 | } 128 | 129 | private void positionBufferForFlushAndRefill(long position) { 130 | assert lib.blockStart(position) == position; 131 | buffer.clear(); // in case limit() < capacity() 132 | filePos = position - buffer.capacity(); 133 | // we only refill when at capacity 134 | buffer.position(buffer.capacity()); 135 | } 136 | 137 | @Override 138 | public long size() { 139 | return fileLength; 140 | } 141 | 142 | 143 | public int readBytes(byte[] dst, int offset, int length) throws IOException { 144 | ensureOpen(); 145 | if (position() > size()) { 146 | throw new EOFException("trying to read at " + position() + " , length is " + size()); 147 | } else if (position() == size()) { 148 | return -1; //SeekableByteChannel contract 149 | } 150 | int total = 0; 151 | if (buffer.remaining() == 0) { 152 | flushAndRefill(); 153 | } 154 | while (buffer.remaining() > 0 && length > buffer.remaining()) { 155 | final int toRead = buffer.remaining(); 156 | total += toRead; 157 | buffer.get(dst, offset, toRead); 158 | offset += toRead; 159 | length -= toRead; 160 | if (length > 0) { 161 | flushAndRefill(); 162 | } 163 | } 164 | if (buffer.remaining() == 0) { // i.e. we're at EOF 165 | return total; 166 | } else { 167 | assert length <= buffer.remaining(); 168 | total += length; 169 | buffer.get(dst, offset, length); 170 | return total; 171 | } 172 | } 173 | 174 | @Override 175 | public int read(ByteBuffer dst) throws IOException { 176 | ensureOpen(); 177 | final int ret = readBytes(dst.array(), dst.position(), dst.remaining()); 178 | dst.position(dst.position() + ret); 179 | return ret; 180 | } 181 | 182 | public int writeBytes(byte[] src, int offset, int length) throws IOException { 183 | ensureOpen(); 184 | ensureWritable(); 185 | int total = 0; 186 | boolean bufferHasMoved = false; 187 | if (buffer.remaining() == 0) { 188 | bufferHasMoved = true; 189 | flushAndForwardBufferWithoutRefill(); 190 | } 191 | while (length > buffer.remaining()) { 192 | final int toWrite = buffer.remaining(); 193 | total += toWrite; 194 | // set blocks which we are about to write to as being dirty 195 | setDirtyBlocksInRange(buffer.position(), buffer.capacity()); 196 | buffer.put(src, offset, toWrite); 197 | offset += toWrite; 198 | length -= toWrite; 199 | bufferHasMoved = true; 200 | flushAndForwardBufferWithoutRefill(); 201 | } 202 | assert length <= buffer.remaining(); 203 | // if we moved the buffer, we need to keep it in sync with the disk 204 | // assuming we don't next overwrite it completely 205 | if (bufferHasMoved && length < buffer.remaining()) { 206 | positionBufferForFlushAndRefill(filePos); 207 | flushAndRefill(); 208 | } 209 | // set blocks which we are about to write to as being dirty 210 | setDirtyBlocksInRange(buffer.position(), buffer.position() + length); 211 | total += length; 212 | buffer.put(src, offset, length); 213 | fileLength = Math.max(fileLength, position()); 214 | return total; 215 | } 216 | 217 | @Override 218 | public int write(ByteBuffer src) throws IOException { 219 | ensureOpen(); 220 | ensureWritable(); 221 | final int ret = writeBytes(src.array(), src.position(), src.remaining()); 222 | src.position(src.position() + ret); 223 | return ret; 224 | } 225 | 226 | public void write(int b) throws IOException { 227 | ensureOpen(); 228 | ensureWritable(); 229 | if (buffer.remaining() == 0) { 230 | flushAndRefill(); 231 | } 232 | dirty[buffer.position() / lib.blockSize()] = true; 233 | buffer.put((byte) b); 234 | fileLength = Math.max(fileLength, position()); 235 | } 236 | 237 | public int read() throws IOException { 238 | ensureOpen(); 239 | if (position() > size()) { 240 | throw new EOFException("trying to read at " + position() + " , length is " + size()); 241 | } else if (position() == size()) { 242 | return -1; //SeekableByteChannel contract 243 | } 244 | if (buffer.remaining() == 0) { 245 | flushAndRefill(); 246 | } 247 | return buffer.get() & 0xFF; 248 | } 249 | 250 | private void flushAndForwardBufferWithoutRefill() throws IOException { 251 | assert buffer.remaining() == 0; 252 | flush(); 253 | filePos += buffer.capacity(); 254 | buffer.clear(); 255 | } 256 | 257 | // sets blocks to dirty if buffer bytes in [start,stop) 258 | // have been written 259 | private void setDirtyBlocksInRange(int start, int stop) { 260 | for (int i=start / lib.blockSize(); i*lib.blockSize() < stop; i++) { 261 | dirty[i] = true; 262 | } 263 | // also set the global dirty bit to true 264 | globalDirty = true; 265 | } 266 | 267 | private void refill() throws IOException { 268 | assert buffer.position() == 0; 269 | if (position() < size()) { 270 | assert lib.blockStart(filePos) == filePos : 271 | "filePos is not a multiple of " + lib.blockSize() + ": filePos=" + filePos; 272 | channel.read(buffer, filePos); 273 | buffer.clear(); 274 | } 275 | } 276 | 277 | private void flushAndRefill() throws IOException { 278 | assert buffer.remaining() == 0; 279 | flushAndForwardBufferWithoutRefill(); 280 | refill(); 281 | } 282 | 283 | @Override 284 | public boolean isOpen() { 285 | return isOpen; 286 | } 287 | 288 | @Override 289 | public ByteChannelAligner truncate(final long size) throws IOException { 290 | ensureOpen(); 291 | ensureWritable(); 292 | flush(); 293 | channel.truncate(size); 294 | fileLength = size; 295 | return this; 296 | } 297 | 298 | public void flush() throws IOException { 299 | ensureOpen(); 300 | if (!globalDirty) { // nothing to do 301 | return; 302 | } 303 | // read only channels cannot get here since there 304 | // will not be any dirty bits 305 | ensureWritable(); 306 | final int oldPos = buffer.position(); 307 | final int oldLim = buffer.limit(); 308 | assert (lib.blockStart(filePos)) == filePos; 309 | for (int i=0; i { 27 | 28 | // TODO (smacke): a builder would be good here 29 | 30 | public DirectIoByteChannelAligner(DirectIoLib lib, 31 | BufferedChannel channel, 32 | AlignedDirectByteBuffer buffer) { 33 | super(lib, channel, buffer); 34 | } 35 | 36 | public static DirectIoByteChannelAligner open(File path) throws IOException { 37 | DirectIoLib lib = DirectIoLib.getLibForPath(path.toString()); 38 | return open(lib, path, lib.defaultBufferSize(), false); 39 | } 40 | 41 | public static DirectIoByteChannelAligner open(File path, int bufferSize) throws IOException { 42 | DirectIoLib lib = DirectIoLib.getLibForPath(path.toString()); 43 | return open(lib, path, bufferSize, false); 44 | } 45 | 46 | public static DirectIoByteChannelAligner open(File path, boolean readOnly) throws IOException { 47 | DirectIoLib lib = DirectIoLib.getLibForPath(path.toString()); 48 | return open(lib, path, lib.defaultBufferSize(), readOnly); 49 | } 50 | 51 | public static DirectIoByteChannelAligner open(File path, int bufferSize, boolean readOnly) throws IOException { 52 | DirectIoLib lib = DirectIoLib.getLibForPath(path.toString()); 53 | return open(lib, path, bufferSize, readOnly); 54 | } 55 | 56 | public static DirectIoByteChannelAligner open(DirectIoLib lib, File path, int bufferSize, boolean readOnly) throws IOException { 57 | if (bufferSize < 0 || (bufferSize % lib.blockSize() != 0)) { 58 | throw new IllegalArgumentException("The buffer capacity must be a multiple of the file system block size"); 59 | } 60 | BufferedChannel channel = DirectIoByteChannel.getChannel(lib, path, readOnly); 61 | AlignedDirectByteBuffer buffer = AlignedDirectByteBuffer.allocate(lib, bufferSize); 62 | return new DirectIoByteChannelAligner(lib, channel, buffer); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/align/MockByteChannelAligner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.align; 17 | 18 | import net.smacke.jaydio.DirectIoLib; 19 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 20 | import net.smacke.jaydio.channel.BufferedChannel; 21 | 22 | public class MockByteChannelAligner extends ByteChannelAligner { 23 | 24 | public MockByteChannelAligner(DirectIoLib lib, 25 | BufferedChannel channel, JaydioByteBuffer buffer) { 26 | super(lib, channel, buffer); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/align/MockDirectIoLib.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.align; 17 | 18 | import net.smacke.jaydio.DirectIoLib; 19 | 20 | public class MockDirectIoLib extends DirectIoLib { 21 | 22 | // package private for testing 23 | 24 | // needed to get a DirectIoLib impl in the 25 | // net.smacke.jaydio.align package, and this 26 | // is a hack to do that. 27 | MockDirectIoLib(int fsBlockSize) { 28 | super(fsBlockSize); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/buffer/AbstractBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | import java.nio.BufferOverflowException; 19 | import java.nio.BufferUnderflowException; 20 | 21 | 22 | 23 | /** 24 | * Skeletal implementation of {@link Buffer}.

25 | * 26 | * @author smacke 27 | * 28 | */ 29 | public abstract class AbstractBuffer implements Buffer { 30 | 31 | protected int position = 0; 32 | protected int limit; 33 | protected int capacity; 34 | protected boolean isOpen; 35 | 36 | protected AbstractBuffer(int pos, int lim, int cap) { 37 | if (cap < 0) { 38 | throw new IllegalArgumentException(); 39 | } 40 | this.capacity = cap; 41 | limit(lim); 42 | position(pos); 43 | this.isOpen = true; 44 | } 45 | 46 | @Override 47 | public final boolean isOpen() { 48 | return isOpen; 49 | } 50 | 51 | @Override 52 | public final int capacity() { 53 | return capacity; 54 | } 55 | 56 | @Override 57 | public AbstractBuffer position(int newPosition) { 58 | if ((newPosition > limit) || (newPosition < 0)) { 59 | throw new IllegalArgumentException(); 60 | } 61 | position = newPosition; 62 | return this; 63 | } 64 | 65 | @Override 66 | public final int position() { 67 | return position; 68 | } 69 | 70 | @Override 71 | public AbstractBuffer limit(int newLimit) { 72 | // enforce the invariants: 73 | // position <= limit <= capacity 74 | if ((newLimit > capacity) || (newLimit < 0)) { 75 | throw new IllegalArgumentException(); 76 | } 77 | limit = newLimit; 78 | this.position = Math.min(this.position, limit); 79 | return this; 80 | } 81 | 82 | @Override 83 | public int limit() { 84 | return limit; 85 | } 86 | 87 | @Override 88 | public AbstractBuffer rewind() { 89 | position = 0; 90 | return this; 91 | } 92 | 93 | 94 | @Override 95 | public AbstractBuffer flip() { 96 | limit = position; 97 | position = 0; 98 | return this; 99 | } 100 | 101 | @Override 102 | public AbstractBuffer clear() { 103 | position = 0; 104 | limit = capacity; 105 | return this; 106 | } 107 | 108 | @Override 109 | public int remaining() { 110 | return limit - position; 111 | } 112 | 113 | @Override 114 | public boolean hasRemaining() { 115 | return position < limit; 116 | } 117 | 118 | 119 | // sanity checking methods 120 | // (package private for use with unit tests) 121 | 122 | 123 | static void checkWithinBounds(int off, int len, int size) { 124 | if ((off | len) < 0) { 125 | throw new IndexOutOfBoundsException(); 126 | } 127 | if (off + len > size) { 128 | throw new IndexOutOfBoundsException(); 129 | } 130 | } 131 | int safeIncrementForGet() { 132 | if (position >= limit) { 133 | throw new BufferUnderflowException(); 134 | } 135 | return position++; 136 | } 137 | 138 | int safeIncrementForPut() { 139 | if (position >= limit) { 140 | throw new BufferOverflowException(); 141 | } 142 | return position++; 143 | } 144 | 145 | @Override 146 | public void close() { 147 | isOpen = false; 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/buffer/AlignedDirectByteBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | import java.nio.BufferOverflowException; 19 | import java.nio.BufferUnderflowException; 20 | import java.nio.ByteBuffer; 21 | 22 | import net.smacke.jaydio.DirectIoLib; 23 | 24 | import com.sun.jna.NativeLong; 25 | import com.sun.jna.Pointer; 26 | import com.sun.jna.ptr.PointerByReference; 27 | 28 | /** 29 | * Implementation of {@link Buffer} which uses JNA to get access to properly aligned 30 | * native memory, for use with the O_DIRECT flag. It is called "AlignedDIRECTByteBuffer" 31 | * after java.nio.DirectByteBuffer, as it uses "direct" memory.

32 | * 33 | * @author smacke 34 | * 35 | */ 36 | public final class AlignedDirectByteBuffer extends AbstractBuffer implements JaydioByteBuffer { 37 | 38 | private Pointer pointer; 39 | private DirectIoLib lib; 40 | 41 | /** 42 | * Allocate capacity bytes of native memory for use as a buffer, and 43 | * return a {@link AlignedDirectByteBuffer} which gives an interface to this memory. The 44 | * memory is allocated with 45 | * {@link DirectIoLib#posix_memalign(PointerByReference,NativeLong,NativeLong) DirectIoLib#posix_memalign()} 46 | * to ensure that the buffer can be used with O_DIRECT. 47 | * 48 | * IT IS VERY IMPORTANT TO CALL {@link #close()} ONCE FINISHED TO FREE MEMORY. 49 | * 50 | * @param capacity The requested number of bytes to allocate 51 | * 52 | * @return A new JnaMemAlignedBuffer of capacity bytes aligned in native memory. 53 | */ 54 | public static AlignedDirectByteBuffer allocate(DirectIoLib lib, int capacity) { 55 | if (capacity % lib.blockSize() > 0) { 56 | throw new IllegalArgumentException("Capacity (" + capacity + ") must be a multiple" 57 | + "of the block size (" + lib.blockSize() + ")"); 58 | } 59 | NativeLong blockSize = new NativeLong(lib.blockSize()); 60 | PointerByReference pointerToPointer = new PointerByReference(); 61 | 62 | // align memory for use with O_DIRECT 63 | DirectIoLib.posix_memalign(pointerToPointer, blockSize, new NativeLong(capacity)); 64 | return new AlignedDirectByteBuffer(lib, pointerToPointer.getValue(), 0, capacity, capacity); 65 | } 66 | 67 | private AlignedDirectByteBuffer(DirectIoLib lib, Pointer pointer, int pos, int lim, int cap) { 68 | super(pos, lim, cap); 69 | this.lib = lib; 70 | this.pointer = pointer; 71 | } 72 | 73 | 74 | 75 | 76 | 77 | @Override 78 | public AlignedDirectByteBuffer get(byte[] dst, int offset, int length) { 79 | checkWithinBounds(offset, length, dst.length); 80 | if (length > remaining()) { 81 | throw new BufferUnderflowException(); 82 | } 83 | pointer.read(position, dst, offset, length); 84 | this.position(position + length); 85 | return this; 86 | } 87 | 88 | @Override 89 | public AlignedDirectByteBuffer get(byte[] dst) { 90 | return get(dst, 0, dst.length); 91 | } 92 | 93 | @Override 94 | public AlignedDirectByteBuffer get(ByteBuffer dst) { 95 | final int length = Math.min(this.remaining(), dst.remaining()); 96 | this.get(dst.array(), dst.position(), length); 97 | dst.position(dst.position()+length); 98 | return this; 99 | } 100 | 101 | 102 | 103 | @Override 104 | public AlignedDirectByteBuffer put(byte[] src, int offset, int length) { 105 | checkWithinBounds(offset, length, src.length); 106 | if (length > remaining()) { 107 | throw new BufferOverflowException(); 108 | } 109 | pointer.write(position, src, offset, length); 110 | this.position(position + length); 111 | return this; 112 | } 113 | 114 | @Override 115 | public AlignedDirectByteBuffer put(byte[] src) { 116 | return this.put(src, 0, src.length); 117 | } 118 | 119 | @Override 120 | public AlignedDirectByteBuffer put(ByteBuffer src) { 121 | final int length = Math.min(this.remaining(), src.remaining()); 122 | this.put(src.array(), src.position(), length); 123 | src.position(src.position()+length); 124 | return this; 125 | } 126 | 127 | 128 | @Override 129 | public byte get() { 130 | return pointer.getByte(safeIncrementForGet()); 131 | } 132 | 133 | @Override 134 | public AlignedDirectByteBuffer put(byte b) { 135 | pointer.setByte(safeIncrementForPut(), b); 136 | return this; 137 | } 138 | 139 | 140 | @Override 141 | public AlignedDirectByteBuffer copy() { 142 | AlignedDirectByteBuffer copy = AlignedDirectByteBuffer.allocate(lib, this.capacity()); 143 | int oldPos = this.position(); 144 | int oldLim = this.limit(); 145 | this.clear(); 146 | byte[] temp = new byte[this.capacity()]; 147 | this.get(temp); 148 | copy.put(temp); 149 | this.position(oldPos); 150 | copy.position(oldPos); 151 | this.limit(oldLim); 152 | copy.limit(oldLim); 153 | return copy; 154 | } 155 | 156 | /** 157 | * @return A view of the native memory which backs this buffer 158 | */ 159 | public Pointer pointer() { 160 | return pointer.share(0); // share view rather than actual pointer 161 | } 162 | 163 | @Override 164 | public void close() { 165 | if (!isOpen) { 166 | return; 167 | } 168 | isOpen = false; 169 | DirectIoLib.free(pointer); // native free 170 | pointer = null; 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/buffer/Buffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | 19 | /** 20 | * A container used to buffer some arbitrary type. The actual storage 21 | * mechanism is flexible.

22 | * 23 | *

The API is similar to that of {@link java.nio.Buffer}, 24 | * but this Buffer is actually extensible and whatnot. 25 | * {@link java.nio.Buffer} has package private methods, preventing its 26 | * use as a superinterface. Also, this buffer impl has no concept of 27 | * a mark like {@link java.nio.Buffer}.

28 | * 29 | * @see java.nio.Buffer 30 | * 31 | * @author smacke 32 | * 33 | */ 34 | public interface Buffer { 35 | 36 | /** 37 | * @see java.nio.Buffer#position() 38 | */ 39 | public int position(); 40 | 41 | 42 | /** 43 | * @see java.nio.Buffer#limit() 44 | * 45 | * @throws java.nio.BufferUnderflowException 46 | * @throws java.nio.BufferOverflowException 47 | */ 48 | public int limit(); 49 | 50 | 51 | /** 52 | * @see java.nio.Buffer#capacity() 53 | */ 54 | public int capacity(); 55 | 56 | 57 | /** 58 | * @see java.nio.Buffer#remaining() 59 | */ 60 | public int remaining(); 61 | 62 | 63 | /** 64 | * @see java.nio.Buffer#hasRemaining() 65 | */ 66 | public boolean hasRemaining(); 67 | 68 | 69 | 70 | 71 | /** 72 | * @see java.nio.Buffer#position(int) 73 | */ 74 | public Buffer position(int newPosition); 75 | 76 | 77 | /** 78 | * @see java.nio.Buffer#limit(int) 79 | */ 80 | public Buffer limit(int newLimit); 81 | 82 | 83 | /** 84 | * @see java.nio.Buffer#clear() 85 | */ 86 | public Buffer clear(); 87 | 88 | 89 | /** 90 | * @see java.nio.Buffer#flip() 91 | */ 92 | public Buffer flip(); 93 | 94 | 95 | /** 96 | * @see java.nio.Buffer#rewind() 97 | */ 98 | public Buffer rewind(); 99 | 100 | 101 | 102 | 103 | /** 104 | * Returns a deep copy of this {@link Buffer}.

105 | */ 106 | public Buffer copy(); 107 | 108 | 109 | 110 | /** 111 | * Free any resources associated with this buffer.

112 | */ 113 | public void close(); 114 | 115 | 116 | 117 | /** 118 | * @return whether the resources associated with this buffer have not yet been freed.

119 | */ 120 | public boolean isOpen(); 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/buffer/JavaHeapByteBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | import java.nio.BufferOverflowException; 19 | import java.nio.BufferUnderflowException; 20 | import java.nio.ByteBuffer; 21 | import java.util.Arrays; 22 | 23 | /** 24 | * A mock implementation of {@link JaydioByteBuffer}, used for testing. 25 | * This class uses Java heap rather than manually malloc'd memory using JNA.

26 | * 27 | * @author smacke 28 | * 29 | */ 30 | public class JavaHeapByteBuffer extends AbstractBuffer implements JaydioByteBuffer { 31 | 32 | private byte[] backing; 33 | 34 | public static JavaHeapByteBuffer allocate(int capacity) { 35 | return new JavaHeapByteBuffer(0, capacity, capacity); 36 | } 37 | 38 | protected JavaHeapByteBuffer(int pos, int lim, int cap) { 39 | super(pos, lim, cap); 40 | backing = new byte[cap]; 41 | } 42 | 43 | @Override 44 | public JavaHeapByteBuffer put(byte[] src) { 45 | return this.put(src, 0, src.length); 46 | } 47 | 48 | @Override 49 | public JavaHeapByteBuffer put(byte[] src, int offset, int length) { 50 | checkWithinBounds(offset, length, src.length); 51 | if (length > remaining()) { 52 | throw new BufferOverflowException(); 53 | } 54 | System.arraycopy(src, offset, backing, position, length); 55 | this.position(position + length); 56 | return this; 57 | } 58 | 59 | @Override 60 | public JavaHeapByteBuffer put(ByteBuffer src) { 61 | final int length = Math.min(this.remaining(), src.remaining()); 62 | this.put(src.array(), src.position(), length); 63 | src.position(src.position()+length); 64 | return this; 65 | } 66 | 67 | @Override 68 | public JavaHeapByteBuffer put(byte b) { 69 | backing[safeIncrementForPut()] = b; 70 | return this; 71 | } 72 | 73 | @Override 74 | public byte get() { 75 | return backing[safeIncrementForGet()]; 76 | } 77 | 78 | @Override 79 | public JavaHeapByteBuffer get(byte[] dst, int offset, int length) { 80 | checkWithinBounds(offset, length, dst.length); 81 | if (length > remaining()) { 82 | throw new BufferUnderflowException(); 83 | } 84 | System.arraycopy(backing, position, dst, offset, length); 85 | this.position(position + length); 86 | return this; 87 | } 88 | 89 | @Override 90 | public JavaHeapByteBuffer get(ByteBuffer dst) { 91 | final int length = Math.min(this.remaining(), dst.remaining()); 92 | this.get(dst.array(), dst.position(), length); 93 | dst.position(dst.position()+length); 94 | return this; 95 | } 96 | 97 | @Override 98 | public JavaHeapByteBuffer get(byte[] dst) { 99 | return this.get(dst, 0, dst.length); 100 | } 101 | 102 | @Override 103 | public JavaHeapByteBuffer copy() { 104 | JavaHeapByteBuffer copy = new JavaHeapByteBuffer(position, limit, capacity); 105 | copy.backing = Arrays.copyOf(backing, backing.length); 106 | return copy; 107 | } 108 | 109 | @Override 110 | public void close() { 111 | if (!isOpen) { 112 | return; 113 | } 114 | isOpen = false; 115 | backing = null; 116 | } 117 | 118 | 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/buffer/JaydioByteBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | public interface JaydioByteBuffer extends Buffer { 19 | 20 | /** 21 | * @see java.nio.ByteBuffer#get() 22 | */ 23 | public byte get(); 24 | 25 | 26 | /** 27 | * @see java.nio.ByteBuffer#get(byte[]) 28 | */ 29 | public JaydioByteBuffer get(byte[] dst); 30 | 31 | 32 | /** 33 | * @see java.nio.ByteBuffer#get(byte[], int, int) 34 | */ 35 | public JaydioByteBuffer get(byte[] dst, int offset, int length); 36 | 37 | /** 38 | * Reads min(this.remaining(),dst.remaining()) bytes from this buffer into dst. 39 | * Note the subtle difference from the corresponding method {@link java.nio.ByteBuffer} -- that 40 | * one does not take the min of the two remaining capacities. 41 | * 42 | * @param dst 43 | * The buffer into which to read. 44 | * 45 | * @return This buffer 46 | */ 47 | public JaydioByteBuffer get(java.nio.ByteBuffer dst); 48 | 49 | 50 | /** 51 | * @see java.nio.ByteBuffer#put(byte) 52 | */ 53 | public JaydioByteBuffer put(byte b); 54 | 55 | 56 | /** 57 | * @see java.nio.ByteBuffer#put(byte[]) 58 | */ 59 | public JaydioByteBuffer put(byte[] src); 60 | 61 | 62 | /** 63 | * @see java.nio.ByteBuffer#put(byte[], int, int) 64 | */ 65 | public JaydioByteBuffer put(byte[] src, int offset, int length); 66 | 67 | 68 | /** 69 | * Writes min(this.remaining(),src.remaining()) bytes into this buffer. 70 | * 71 | * @param src 72 | * The ByteBuffer from which to write into this buffer 73 | * 74 | * @return This buffer 75 | */ 76 | public JaydioByteBuffer put(java.nio.ByteBuffer src); 77 | 78 | 79 | // override return type 80 | @Override 81 | public JaydioByteBuffer copy(); 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/channel/BufferedChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | import java.io.IOException; 19 | import java.nio.channels.Channel; 20 | 21 | import net.smacke.jaydio.buffer.Buffer; 22 | 23 | 24 | /** 25 | * Channel which supports positional {@link #read(Buffer,long) read}s and 26 | * {@link #write(Buffer, long) write}s from/to a buffer to an underlying resource. 27 | * 28 | * @author smacke 29 | * 30 | */ 31 | public interface BufferedChannel extends Channel { 32 | 33 | 34 | /** 35 | * Writes from the src buffer into this channel at position.

36 | * 37 | * @param src 38 | * The {@link Buffer} to write from 39 | * 40 | * @param position 41 | * The position within the file at which to start writing 42 | * 43 | * @return How many bytes were written from src into the file 44 | * @throws IOException 45 | */ 46 | public int write(T src, long position) throws IOException; 47 | 48 | 49 | 50 | 51 | /** 52 | * Reads from this channel into the dst buffer from position.

53 | * 54 | * @param dst 55 | * The {@link Buffer} to read into 56 | * 57 | * @param position 58 | * The position within the file at which to start reading 59 | * 60 | * @return How many bytes were placed into dst 61 | * @throws IOException 62 | */ 63 | public int read(T dst, long position) throws IOException; 64 | 65 | 66 | 67 | /** 68 | * @return The file size for this channel 69 | */ 70 | public long size(); 71 | 72 | 73 | /** 74 | * @return true if this channel is read only, false otherwise 75 | */ 76 | public boolean isReadOnly(); 77 | 78 | 79 | /** 80 | * Truncates this file's length to fileLength.

81 | * 82 | * @param fileLength The length to which to truncate 83 | * 84 | * @return This UnsafeByteAlignedChannel 85 | * 86 | * @throws IOException 87 | */ 88 | public BufferedChannel truncate(long fileLength) throws IOException; 89 | 90 | 91 | /** 92 | * @return The file descriptor for this channel 93 | */ 94 | public int getFD(); 95 | } -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/channel/DirectIoByteChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.nio.channels.ClosedChannelException; 21 | import java.nio.channels.NonWritableChannelException; 22 | 23 | import net.smacke.jaydio.DirectIoLib; 24 | import net.smacke.jaydio.align.ByteChannelAligner; 25 | import net.smacke.jaydio.buffer.AlignedDirectByteBuffer; 26 | import net.smacke.jaydio.buffer.JavaHeapByteBuffer; 27 | 28 | /** 29 | * An {@link BufferedChannel} implementation which uses {@link DirectIoLib} 30 | * for JNA hooks to native Linux methods. Particular, the O_DIRECT flag is used.

31 | * 32 | *

One might wonder why the functionality in this class is not directly subsumed by 33 | * {@link ByteChannelAligner}. For testing purposes, it made sense to separate out the 34 | * alignment logic from the actual I/O logic. For example, it is possible to test the 35 | * alignment logic completely in-memory using {@link MockByteChannel}s and 36 | * {@link JavaHeapByteBuffer}s thanks to this abstraction.

37 | * 38 | * @author smacke 39 | * 40 | */ 41 | public final class DirectIoByteChannel implements BufferedChannel { 42 | 43 | private DirectIoLib lib; 44 | private int fd; 45 | private boolean isOpen; 46 | private long fileLength; 47 | private boolean isReadOnly; 48 | 49 | public static DirectIoByteChannel getChannel(File file, boolean readOnly) throws IOException { 50 | DirectIoLib lib = DirectIoLib.getLibForPath(file.toString()); 51 | return getChannel(lib, file, readOnly); 52 | } 53 | 54 | public static DirectIoByteChannel getChannel(DirectIoLib lib, File file, boolean readOnly) throws IOException { 55 | int fd = lib.oDirectOpen(file.toString(), readOnly); 56 | long length = file.length(); 57 | return new DirectIoByteChannel(lib, fd, length, readOnly); 58 | } 59 | 60 | private DirectIoByteChannel(DirectIoLib lib, int fd, long fileLength, boolean readOnly) { 61 | this.lib = lib; 62 | this.fd = fd; 63 | this.isOpen = true; 64 | this.isReadOnly = readOnly; 65 | this.fileLength = fileLength; 66 | } 67 | 68 | private void ensureOpen() throws ClosedChannelException { 69 | if (!isOpen()) { 70 | throw new ClosedChannelException(); 71 | } 72 | } 73 | 74 | private void ensureWritable() { 75 | if (isReadOnly()) { 76 | throw new NonWritableChannelException(); 77 | } 78 | } 79 | 80 | 81 | @Override 82 | public int read(AlignedDirectByteBuffer dst, long position) throws IOException { 83 | ensureOpen(); 84 | return lib.pread(fd, dst, position); 85 | } 86 | 87 | @Override 88 | public int write(AlignedDirectByteBuffer src, long position) throws IOException { 89 | ensureOpen(); 90 | ensureWritable(); 91 | assert src.position() == lib.blockStart(src.position()); 92 | 93 | int written = lib.pwrite(fd, src, position); 94 | 95 | // update file length if we wrote past it 96 | fileLength = Math.max(position + written, fileLength); 97 | return written; 98 | } 99 | 100 | @Override 101 | public DirectIoByteChannel truncate(final long length) throws IOException { 102 | ensureOpen(); 103 | ensureWritable(); 104 | if (DirectIoLib.ftruncate(fd, length) < 0) { 105 | throw new IOException("Error during truncate on descriptor " + fd + ": " + 106 | DirectIoLib.getLastError()); 107 | } 108 | fileLength = length; 109 | return this; 110 | } 111 | 112 | @Override 113 | public long size() { 114 | return fileLength; 115 | } 116 | 117 | @Override 118 | public int getFD() { 119 | return fd; 120 | } 121 | 122 | 123 | @Override 124 | public boolean isOpen() { 125 | return isOpen; 126 | } 127 | 128 | @Override 129 | public boolean isReadOnly() { 130 | return isReadOnly; 131 | } 132 | 133 | @Override 134 | public void close() throws IOException { 135 | if (!isOpen()) { 136 | return; 137 | } 138 | try { 139 | if (!isReadOnly()) { 140 | truncate(fileLength); 141 | } 142 | } finally { 143 | isOpen = false; 144 | if (lib.close(fd) < 0) { 145 | throw new IOException("Error closing file with descriptor " + fd + ": " + 146 | DirectIoLib.getLastError()); 147 | } 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/net/smacke/jaydio/channel/MockByteChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | import java.io.IOException; 19 | import java.nio.channels.ClosedChannelException; 20 | import java.nio.channels.NonWritableChannelException; 21 | 22 | import net.smacke.jaydio.DirectIoLib; 23 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 24 | 25 | 26 | /** 27 | * A mock implementation of {@link BufferedChannel}, to be used for testing. 28 | * The mock channel is backed {@link #file}, located on java heap.

29 | * 30 | * @author smacke 31 | * 32 | */ 33 | public final class MockByteChannel implements BufferedChannel { 34 | 35 | private long fileLength; 36 | private boolean isOpen; 37 | private boolean isReadOnly; 38 | 39 | // keep the underlying "file" in-memory 40 | // allows for lightweight mock testing 41 | private byte[] file; 42 | 43 | /** 44 | * Factory method returning a new {@link MockByteChannel}.

45 | * 46 | * @param length 47 | * The maximum length that the "file" will grow to -- 48 | * this is how many bytes will be allocated on Java heap 49 | * 50 | * @param blockSize 51 | * The block size of the fake file system for this fake file 52 | * 53 | * @param readOnly 54 | * Whether this "channel" is in readOnly mode 55 | * 56 | * @return A new {@link MockByteChannel} with the corresponding parameters 57 | */ 58 | public static MockByteChannel getChannel(long length, int blockSize, boolean readOnly) { 59 | return new MockByteChannel(DirectIoLib.blockEnd(blockSize, length), readOnly); 60 | } 61 | 62 | protected MockByteChannel(long fileLength, boolean readOnly) { 63 | this.isReadOnly = readOnly; 64 | this.isOpen = true; 65 | this.file = new byte[(int)fileLength]; 66 | this.fileLength = readOnly ? fileLength : 0; 67 | } 68 | 69 | @Override 70 | public int read(JaydioByteBuffer dst, long position) throws ClosedChannelException { 71 | ensureOpen(); 72 | dst.clear(); 73 | int endPos = Math.min((int)position + dst.limit(), (int)size()); 74 | for (int i = (int)position; i < endPos; i++) { 75 | dst.put(file[i]); 76 | } 77 | dst.clear(); 78 | return (int)(endPos - position); 79 | } 80 | 81 | @Override 82 | public int write(JaydioByteBuffer src, long position) throws IOException { 83 | ensureOpen(); 84 | ensureWritable(); 85 | int oldPos = src.position(); 86 | src.position(0); 87 | 88 | for (int i = (int)position; i < position + src.limit(); i++) { 89 | file[i] = src.get(); 90 | } 91 | 92 | // If write past current EOF, update the file length. 93 | fileLength = Math.max(position + src.limit(), fileLength); 94 | src.position(oldPos); 95 | return src.limit(); 96 | } 97 | 98 | private void ensureOpen() throws ClosedChannelException { 99 | if (!isOpen) { 100 | throw new ClosedChannelException(); 101 | } 102 | } 103 | 104 | private void ensureWritable() throws NonWritableChannelException { 105 | if (isReadOnly) { 106 | throw new NonWritableChannelException(); 107 | } 108 | } 109 | 110 | @Override 111 | public boolean isOpen() { 112 | return isOpen; 113 | } 114 | 115 | @Override 116 | public boolean isReadOnly() { 117 | return isReadOnly; 118 | } 119 | 120 | @Override 121 | public long size() { 122 | return fileLength; 123 | } 124 | 125 | @Override 126 | public MockByteChannel truncate(long fileLength) 127 | throws IOException { 128 | ensureOpen(); 129 | ensureWritable(); 130 | this.fileLength = fileLength; 131 | return this; 132 | } 133 | 134 | @Override 135 | public int getFD() { 136 | throw new UnsupportedOperationException("mock channel is not backed by file"); 137 | } 138 | 139 | @Override 140 | public void close() throws IOException { 141 | isOpen = false; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/align/TestAlignedIO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.align; 17 | 18 | import java.io.IOException; 19 | import java.util.Arrays; 20 | import java.util.Random; 21 | 22 | import net.smacke.jaydio.DirectIoLib; 23 | import net.smacke.jaydio.buffer.JavaHeapByteBuffer; 24 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 25 | import net.smacke.jaydio.channel.BufferedChannel; 26 | import net.smacke.jaydio.channel.MockByteChannel; 27 | 28 | import org.junit.Assert; 29 | import org.junit.Before; 30 | import org.junit.Test; 31 | 32 | 33 | /** 34 | * Tests which use MockJnaBuffers and MockJnaChannels to fuzz 35 | * DirectIndexInputs/Outputs, which will do buffered "I/O" totally 36 | * in-memory, but in an aligned fashion. These tests hit most of the 37 | * nasty error-prone code paths but are quite fast.

38 | * 39 | * @author smacke 40 | * 41 | */ 42 | public class TestAlignedIO extends Assert { 43 | 44 | private int smallWriteSize; 45 | private int seekTrials; 46 | private long randomSeed; 47 | private int alignedFileSize; 48 | private int unalignedFileSize; 49 | private int bufferSize; 50 | private DirectIoLib mockLib; 51 | 52 | private Random rand; 53 | 54 | 55 | public TestAlignedIO() { 56 | this.smallWriteSize = 42; 57 | this.seekTrials = 10; // number of times to seek in tests which do so 58 | this.randomSeed = 42; // constant so that the fuzzer does the same thing each time 59 | this.alignedFileSize = 1024*2; // chosen so that FILE_SIZE % 512 == 0 60 | this.unalignedFileSize = this.alignedFileSize + 217; // chosen so that FILE_SIZE_UNALIGNED is not a power of 2 61 | int blockSize = 4096; 62 | this.bufferSize = 2*blockSize; 63 | this.mockLib = new MockDirectIoLib(blockSize); 64 | } 65 | 66 | @Before 67 | public void setUp() { 68 | // so that it doesn't matter what order tests execute in 69 | rand = new Random(randomSeed); 70 | } 71 | 72 | 73 | @Test 74 | public void testWritingWithoutSeeking() throws IOException { 75 | performWriteTest(unalignedFileSize, false, false); 76 | performWriteTest(alignedFileSize, false, false); 77 | } 78 | 79 | @Test 80 | public void testSeekingWithSmallWrites() throws IOException { 81 | performWriteTest(unalignedFileSize, true, true); 82 | performWriteTest(alignedFileSize, true, true); 83 | } 84 | 85 | @Test 86 | public void testSeekingWithLargeWrites() throws IOException { 87 | performWriteTest(unalignedFileSize, false, true); 88 | performWriteTest(alignedFileSize, false, true); 89 | } 90 | 91 | 92 | private void performWriteTest(final int fileSize, final boolean smallWrites, final boolean seek) throws IOException { 93 | // the gold standard, what we expect to see 94 | byte[] gold = getGoldBytes(fileSize); 95 | 96 | // "write" bytes to the mock channel 97 | BufferedChannel channel = MockByteChannel.getChannel(fileSize, mockLib.blockSize(), false); 98 | MockByteChannelAligner aligned = getMockAlignedChannel(mockLib, channel, bufferSize, fileSize); 99 | aligned.writeBytes(gold, 0, fileSize); 100 | 101 | if (seek) { 102 | // now seek around a bit and write random bytes 103 | for (int i=0; i channel = MockByteChannel.getChannel(fileSize, mockLib.blockSize(), false); 129 | MockByteChannelAligner aligned = getMockAlignedChannel(mockLib, channel, bufferSize, fileSize); 130 | aligned.writeBytes(gold, 0, fileSize); 131 | 132 | aligned.position(0); 133 | aligned.writeBytes(gold, 0, fileSize); 134 | 135 | // second flush should do nothing 136 | aligned.flush(); 137 | aligned.flush(); 138 | 139 | assertEquals(fileSize, aligned.size()); 140 | 141 | aligned.truncate(fileSize); 142 | checkConsistency(mockLib, channel, bufferSize, gold, fileSize); 143 | // underlying channel gets closed in checkConsistency 144 | } 145 | 146 | @Test 147 | public void testStreamReadingAndWriting() throws IOException { 148 | // unaligned file size less than one buffer long 149 | int fileSize = bufferSize/2 + bufferSize/4; 150 | 151 | byte[] gold = getGoldBytes(fileSize); 152 | 153 | 154 | // "write" bytes to the mock channel 155 | BufferedChannel channel = MockByteChannel.getChannel(fileSize, mockLib.blockSize(), false); 156 | MockByteChannelAligner aligned = getMockAlignedChannel(mockLib, channel, bufferSize, fileSize); 157 | 158 | // copy gold to file 159 | aligned.writeBytes(gold, 0, fileSize); 160 | aligned.position(0); 161 | 162 | // seek to start of file 163 | aligned.position(0); 164 | 165 | // now go stream through the file, alternating 166 | // between reading and writing. 167 | int remaining = fileSize; 168 | int position = 0; 169 | boolean reading = false; 170 | while (remaining > 0) { 171 | final int forward = 1+rand.nextInt(remaining); 172 | remaining -= forward; 173 | byte[] forwardBuf = new byte[forward]; 174 | if (reading) { 175 | // don't actually do anything; just move file pointer forward 176 | // using read semantics 177 | aligned.readBytes(forwardBuf, 0, forward); 178 | } else { 179 | rand.nextBytes(forwardBuf); 180 | aligned.writeBytes(forwardBuf, 0, forward); 181 | System.arraycopy(forwardBuf, 0, gold, position, forward); 182 | } 183 | position += forward; 184 | reading = !reading; 185 | } 186 | 187 | assertEquals(fileSize, aligned.size()); 188 | 189 | aligned.truncate(fileSize); 190 | checkConsistency(mockLib, channel, bufferSize, gold, fileSize); 191 | // underlying channel gets closed in checkConsistency 192 | } 193 | 194 | private static MockByteChannelAligner getMockAlignedChannel(DirectIoLib mockLib, BufferedChannel channel, int bufferSize, long fileSize) throws IOException { 195 | JaydioByteBuffer buffer = JavaHeapByteBuffer.allocate(bufferSize); 196 | return new MockByteChannelAligner(mockLib, channel, buffer); 197 | } 198 | 199 | private static MockByteChannelAligner getMockAlignedChannel(DirectIoLib lib, BufferedChannel channel, int bufferSize) { 200 | JaydioByteBuffer buffer = JavaHeapByteBuffer.allocate(bufferSize); 201 | return new MockByteChannelAligner(lib, channel, buffer); 202 | } 203 | 204 | private byte[] getGoldBytes(int fileSize) { 205 | byte[] expected = new byte[fileSize]; 206 | rand.nextBytes(expected); 207 | return expected; 208 | } 209 | 210 | /* Make sure we wrote the same stuff 211 | * 212 | * NOTE: In general it does not make sense of the same ByteAlignedChannels to 213 | * share the same underlying UnsafeByteAlignedChannels. This is because of the 214 | * internal bookkeeping that the safe channel does to keep track of length. The 215 | * only reason this works is because we instantiate the safe channel after the 216 | * bytes have been flushed to the unsafe channel -- the unsafe channel will think 217 | * that the file length is longer than it actually will eventually end up being, 218 | * so the safe channel that we instantiate here doesn't think we're reading past 219 | * EOF when we get the bytes we're reading. 220 | */ 221 | private static void checkConsistency(DirectIoLib mockLib, BufferedChannel channel, int bufferSize, byte[] gold, int fileSize) throws IOException { 222 | // this is where we'll read back in from the files 223 | byte[] directRead = new byte[fileSize]; 224 | 225 | // Now make sure we didn't fail 226 | // (use same channel as the one we wrote to; it stored the bytes in RAM) 227 | MockByteChannelAligner aligned = getMockAlignedChannel(mockLib, channel, bufferSize); 228 | aligned.readBytes(directRead, 0, fileSize); 229 | aligned.close(); 230 | 231 | // make sure the byte arrays are the same 232 | assertTrue(Arrays.equals(gold, directRead)); 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/align/TestDirectIO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.align; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.io.RandomAccessFile; 21 | import java.util.Arrays; 22 | import java.util.Random; 23 | 24 | import net.smacke.jaydio.DirectIoLib; 25 | import net.smacke.jaydio.align.DirectIoByteChannelAligner; 26 | 27 | import org.junit.Assert; 28 | import org.junit.Test; 29 | 30 | 31 | /** 32 | * Test class for direct I/O. It is very easy to make mistakes with block aligned I/O, 33 | * so these tests actually fuzz the implementation to try and break it.

34 | * 35 | *

36 | * Because this class does I/O, and because it behaves nondeterministically, 37 | * it can be rather slow. Use {@link TestAlignedIO} if 38 | * you need a test which runs quickly but still hits most of the error-prone 39 | * code paths.

40 | * 41 | * @author smacke 42 | * 43 | */ 44 | public class TestDirectIO extends Assert { 45 | 46 | public static final String DIRECT_FILE_NAME = "direct_file"; 47 | public static final String GOLD_FILE_NAME = "gold_file"; 48 | 49 | private int smallWriteSize; 50 | private int seekTrials; 51 | private int alignedFileSize; 52 | private int unalignedFileSize; 53 | private int bufferSize; 54 | private DirectIoLib lib; 55 | 56 | public TestDirectIO() { 57 | this.smallWriteSize = 42; 58 | this.seekTrials = 100; // number of times to seek in tests which do so 59 | this.alignedFileSize = 1024*128; // chosen so that FILE_SIZE % 512 == 0 60 | this.unalignedFileSize = this.alignedFileSize + 217; // chosen so that FILE_SIZE_UNALIGNED is not a power of 2 61 | this.lib = DirectIoLib.getLibForPath(System.getProperty("java.io.tmpdir")); // since everything goes in /tmp 62 | this.bufferSize = 8*lib.blockSize(); 63 | } 64 | 65 | private static File getTempDirectory(String prefix, String suffix) throws IOException { 66 | File temp = File.createTempFile(prefix, suffix); 67 | if(!(temp.delete())) { 68 | throw new IOException("Could not delete temp file: " + temp.getAbsolutePath()); 69 | } else if (!temp.mkdir()) { 70 | throw new IOException("could not create temporary directory"); 71 | } 72 | return temp; 73 | } 74 | 75 | @Test 76 | public void testWritingWithoutSeeking() throws IOException { 77 | performWriteTest(unalignedFileSize, false, false); 78 | performWriteTest(alignedFileSize, false, false); 79 | } 80 | 81 | @Test 82 | public void testSeekingWithSmallEdits() throws IOException { 83 | performWriteTest(unalignedFileSize, true, true); 84 | performWriteTest(alignedFileSize, true, true); 85 | } 86 | 87 | @Test 88 | public void testSeekingWithLargeEdits() throws IOException { 89 | performWriteTest(unalignedFileSize, true, false); 90 | performWriteTest(alignedFileSize, true, false); 91 | } 92 | 93 | 94 | private void performWriteTest(final int fileSize, final boolean seek, final boolean smallWrites) throws IOException { 95 | File temp = getTempDirectory("temp", Long.toString(System.nanoTime())); 96 | 97 | // use a RandomAccessFile as the gold standard 98 | // against which to compare the direct I/O channel. 99 | File goldFile = new File(temp, GOLD_FILE_NAME); 100 | RandomAccessFile gold = new RandomAccessFile(goldFile, "rw"); 101 | 102 | // channel for direct i/o 103 | File directFile = new File(temp, DIRECT_FILE_NAME); 104 | DirectIoByteChannelAligner direct = DirectIoByteChannelAligner.open(lib, directFile, bufferSize, false); 105 | 106 | try { 107 | Random rand = new Random(System.nanoTime()); 108 | byte[] writeBuffer = new byte[fileSize]; 109 | rand.nextBytes(writeBuffer); 110 | 111 | // write random bytes to each file 112 | direct.writeBytes(writeBuffer, 0, fileSize); 113 | gold.write(writeBuffer); 114 | 115 | if (seek) { 116 | // now seek around a bit and write random bytes 117 | for (int i=0; i 0) { 238 | final int forward = 1+rand.nextInt(remaining); 239 | remaining -= forward; 240 | byte[] forwardBuf = new byte[forward]; 241 | if (reading) { 242 | direct.readBytes(forwardBuf, 0, forward); 243 | gold.read(forwardBuf, 0, forward); 244 | } else { 245 | rand.nextBytes(forwardBuf); 246 | direct.writeBytes(forwardBuf, 0, forward); 247 | gold.write(forwardBuf, 0, forward); 248 | } 249 | reading = !reading; 250 | } 251 | 252 | checkConsistency(direct, gold, directFile, fileSize); 253 | } 254 | finally { 255 | // cleanup 256 | gold.close(); 257 | direct.close(); 258 | goldFile.delete(); 259 | directFile.delete(); 260 | if (!temp.delete()) { 261 | throw new IOException("Error: could not delete temp directory " + temp.getAbsolutePath()); 262 | } 263 | } 264 | } 265 | 266 | // Make sure we wrote the same stuff 267 | // (assuming direct input works) 268 | private static void checkConsistency(DirectIoByteChannelAligner direct, RandomAccessFile gold, File directFile, int fileSize) throws IOException { 269 | // files written should have same lengths 270 | assertEquals(gold.length(), direct.size()); 271 | 272 | // this is where we'll read back in from the files 273 | byte[] goldRead = new byte[fileSize]; 274 | byte[] directRead = new byte[fileSize]; 275 | 276 | direct.position(0); 277 | direct.readBytes(directRead, 0, fileSize); 278 | gold.getChannel().position(0); 279 | gold.read(goldRead, 0, fileSize); 280 | 281 | // make sure files are the same 282 | assertTrue(Arrays.equals(goldRead, directRead)); 283 | 284 | Arrays.fill(directRead, (byte)11); 285 | DirectIoByteChannelAligner directNew = null; 286 | try { 287 | direct.truncate(direct.size()); 288 | directNew = DirectIoByteChannelAligner.open(directFile); 289 | assertEquals(gold.length(), directNew.size()); 290 | directNew.readBytes(directRead, 0, fileSize); 291 | // make sure files are the same 292 | assertTrue(Arrays.equals(goldRead, directRead)); 293 | } finally { 294 | if (directNew != null) { 295 | directNew.close(); 296 | } 297 | } 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/buffer/AbstractBufferTester.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | import java.nio.BufferOverflowException; 19 | import java.nio.BufferUnderflowException; 20 | 21 | import org.junit.After; 22 | import org.junit.Assert; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | 27 | /** 28 | * Abstract base class to test various implementations of {@link Buffer}. 29 | * 30 | * @author smacke 31 | * 32 | */ 33 | public abstract class AbstractBufferTester extends Assert { 34 | 35 | protected JaydioByteBuffer buffer; 36 | 37 | protected abstract JaydioByteBuffer createInstance(); 38 | 39 | @Before 40 | public void setUp() { 41 | buffer = createInstance(); 42 | } 43 | 44 | @After 45 | public void tearDown() { 46 | buffer.close(); 47 | } 48 | 49 | @Test 50 | public void testGetAdvancesPosition() { 51 | assert buffer.position() < buffer.limit(); 52 | 53 | int oldPos = buffer.position(); 54 | buffer.get(); 55 | 56 | assertEquals(buffer.position(), oldPos+1); 57 | } 58 | 59 | @Test 60 | public void testPutAdvancesPosition() { 61 | assert buffer.position() < buffer.limit(); 62 | 63 | int oldPos = buffer.position(); 64 | buffer.put((byte)1); 65 | 66 | assertEquals(buffer.position(), oldPos+1); 67 | } 68 | 69 | @Test 70 | public void testBulkGetAdvancesPosition() { 71 | int size = 5; 72 | assert buffer.position() + size < buffer.limit(); 73 | byte[] dummy = new byte[size]; 74 | int oldPos = buffer.position(); 75 | buffer.put(dummy, 0, size); 76 | assertEquals(oldPos + size, buffer.position()); 77 | } 78 | 79 | @Test 80 | public void testBulkPutAdvancesPosition() { 81 | int size = 5; 82 | assert buffer.position() + size < buffer.limit(); 83 | byte[] dummy = new byte[size]; 84 | int oldPos = buffer.position(); 85 | buffer.get(dummy, 0, size); 86 | assertEquals(oldPos + size, buffer.position()); 87 | } 88 | 89 | @Test(expected = IndexOutOfBoundsException.class) 90 | public void testReadPastArrayBoundFails() { 91 | int size = 10; 92 | assert buffer.position()+size < buffer.limit(); 93 | byte[] anything = new byte[10]; 94 | buffer.get(anything, size/2, size); 95 | } 96 | 97 | @Test(expected = IndexOutOfBoundsException.class) 98 | public void testWritePastArrayBoundFails() { 99 | int size = 10; 100 | assert buffer.position()+size < buffer.limit(); 101 | byte[] anything = new byte[10]; 102 | buffer.put(anything, size/2, size); 103 | } 104 | 105 | @Test(expected = BufferUnderflowException.class) 106 | public void testReadPastBufferLimitUnderflows() { 107 | int size = buffer.limit() - buffer.position() + 7; 108 | byte[] dummy = new byte[size]; 109 | buffer.get(dummy); 110 | } 111 | 112 | @Test(expected = BufferOverflowException.class) 113 | public void testWritePastBufferLimitOverflows() { 114 | int size = buffer.limit() - buffer.position() + 7; 115 | byte[] dummy = new byte[size]; 116 | buffer.put(dummy); 117 | } 118 | 119 | @Test 120 | public void testPosition() { 121 | int pos = 5; 122 | buffer.position(pos); 123 | assertEquals(pos, buffer.position()); 124 | } 125 | 126 | @Test 127 | public void testLimit() { 128 | int lim = 5; 129 | buffer.limit(lim); 130 | assertEquals(lim, buffer.limit()); 131 | } 132 | 133 | @Test 134 | public void testFlip() { 135 | int oldPos = buffer.position(); 136 | buffer.flip(); 137 | assertEquals(oldPos, buffer.limit()); 138 | assertEquals(0, buffer.position()); 139 | } 140 | 141 | @Test 142 | public void testClear() { 143 | buffer.clear(); 144 | assertEquals(buffer.capacity(), buffer.limit()); 145 | assertEquals(0, buffer.position()); 146 | } 147 | 148 | @Test 149 | public void testCopyIsIndependent() { 150 | JaydioByteBuffer copy = buffer.copy(); 151 | try { 152 | buffer.clear(); 153 | copy.clear(); 154 | buffer.put((byte)1); 155 | assertNotEquals(buffer.position(), copy.position()); 156 | copy.put((byte)2); 157 | buffer.flip(); 158 | copy.flip(); 159 | byte bufferByte = buffer.get(); 160 | byte copyByte = copy.get(); 161 | assertNotEquals(bufferByte, copyByte); 162 | } finally { 163 | copy.close(); 164 | } 165 | } 166 | } -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/buffer/TestAlignedDirectByteBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | import net.smacke.jaydio.DirectIoLib; 19 | import net.smacke.jaydio.buffer.AlignedDirectByteBuffer; 20 | 21 | import org.junit.BeforeClass; 22 | 23 | /** 24 | * Concrete {@link AbstractBufferTester} used to test 25 | * {@link AlignedDirectByteBuffer}. This requires access to native 26 | * memory and will only work on Linux.

27 | * 28 | * @author smacke 29 | * 30 | */ 31 | public class TestAlignedDirectByteBuffer extends AbstractBufferTester { 32 | 33 | private static DirectIoLib lib; 34 | 35 | @BeforeClass public static void setupClass() { 36 | lib = DirectIoLib.getLibForPath(System.getProperty("java.io.tmpdir")); 37 | } 38 | 39 | @Override 40 | protected AlignedDirectByteBuffer createInstance() { 41 | return AlignedDirectByteBuffer.allocate(lib, lib.defaultBufferSize()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/buffer/TestJavaHeapByteBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.buffer; 17 | 18 | public class TestJavaHeapByteBuffer extends AbstractBufferTester { 19 | 20 | @Override 21 | protected JaydioByteBuffer createInstance() { 22 | return JavaHeapByteBuffer.allocate(500); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/channel/BufferedChannelAbstractTester.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | 19 | import java.io.IOException; 20 | import java.nio.channels.NonWritableChannelException; 21 | import java.util.Arrays; 22 | 23 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 24 | import net.smacke.jaydio.channel.BufferedChannel; 25 | 26 | import org.junit.After; 27 | import org.junit.Assert; 28 | import org.junit.Before; 29 | import org.junit.Test; 30 | 31 | /** 32 | * @author smacke 33 | * 34 | */ 35 | public abstract class BufferedChannelAbstractTester extends Assert { 36 | 37 | protected BufferedChannel channel; 38 | protected T buffer; 39 | protected long testPosition; 40 | 41 | @Before 42 | public abstract void setUp() throws IOException; 43 | 44 | @After 45 | public void tearDown() throws IOException { 46 | channel.close(); 47 | buffer.close(); 48 | } 49 | 50 | @Test 51 | public void testClosedChannelIsNotOpen() throws IOException { 52 | channel.close(); 53 | assertFalse(channel.isOpen()); 54 | } 55 | 56 | @Test(expected = NonWritableChannelException.class) 57 | public void testCannotWriteToReadOnlyChannel() throws IOException { 58 | assert channel.isReadOnly(); 59 | 60 | channel.write(buffer, testPosition); 61 | } 62 | 63 | @Test 64 | public void testWritesAreActuallyWritten() throws IOException { 65 | int size = 7; 66 | assert size < buffer.capacity(); 67 | byte[] written = new byte[size]; 68 | for (int i=0; i channel.size(); 85 | channel.write(buffer, testPosition); 86 | assertEquals(expected, channel.size()); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/channel/TestDirectIoByteChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.nio.channels.NonWritableChannelException; 21 | import java.util.Arrays; 22 | 23 | import net.smacke.jaydio.DirectIoLib; 24 | import net.smacke.jaydio.buffer.AlignedDirectByteBuffer; 25 | import net.smacke.jaydio.channel.DirectIoByteChannel; 26 | 27 | import org.junit.After; 28 | import org.junit.Before; 29 | import org.junit.BeforeClass; 30 | import org.junit.Test; 31 | 32 | /** 33 | * Test class which extends {@link BufferedChannelAbstractTester}. 34 | * This does actual file I/O with the O_DIRECT flag (through JNA).

35 | * 36 | * @author smacke 37 | * 38 | */ 39 | public class TestDirectIoByteChannel extends BufferedChannelAbstractTester { 40 | 41 | private File tempDir; 42 | private File tempFile; 43 | private static DirectIoLib lib; 44 | 45 | private static File getTempDirectory(String prefix, String suffix) throws IOException { 46 | File temp = File.createTempFile(prefix, suffix); 47 | if(!(temp.delete())) { 48 | throw new IOException("Could not delete temp file: " + temp.getAbsolutePath()); 49 | } else if (!temp.mkdir()) { 50 | throw new IOException("could not create temporary directory"); 51 | } 52 | return temp; 53 | } 54 | 55 | @BeforeClass public static void setupClass() { 56 | lib = DirectIoLib.getLibForPath(System.getProperty("java.io.tmpdir")); 57 | } 58 | 59 | 60 | private void subSetup(boolean readOnly) throws IOException { 61 | channel = DirectIoByteChannel.getChannel(tempFile, readOnly); 62 | buffer = AlignedDirectByteBuffer.allocate(lib, 2*lib.blockSize()); 63 | testPosition = lib.blockSize(); 64 | int startFilelength = 2*lib.blockSize(); 65 | byte[] fileContents = new byte[startFilelength]; 66 | Arrays.fill(fileContents, (byte)7); 67 | buffer.put(fileContents); 68 | channel.write(buffer, 0); 69 | buffer.clear(); 70 | } 71 | 72 | @Override 73 | @Before public void setUp() throws IOException { 74 | tempDir = getTempDirectory("temp", Long.toString(System.nanoTime())); 75 | tempFile = new File(tempDir, "channel"); 76 | subSetup(false); 77 | } 78 | 79 | @Override 80 | @After 81 | public void tearDown() throws IOException { 82 | super.tearDown(); 83 | tempFile.delete(); 84 | if (!tempDir.delete()) { 85 | throw new IOException("could not delete temp directory " + tempDir.getAbsolutePath()); 86 | } 87 | } 88 | 89 | @Override 90 | @Test(expected = NonWritableChannelException.class) 91 | public void testCannotWriteToReadOnlyChannel() throws IOException { 92 | super.tearDown(); // super method doesn't delete file 93 | subSetup(true); 94 | super.testCannotWriteToReadOnlyChannel(); 95 | } 96 | 97 | public void testCannotOpenNewFileInReadOnlyMode() throws IOException { 98 | tearDown(); // delete temp file 99 | try{ 100 | subSetup(true); 101 | fail("trying to open file in read only mode should throw exception"); 102 | } catch (IOException expected) { 103 | // expected 104 | } 105 | setUp(); // recreate temp file 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/net/smacke/jaydio/channel/TestMockByteChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2014 Stephen Macke (smacke@cs.stanford.edu) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package net.smacke.jaydio.channel; 17 | 18 | import java.io.IOException; 19 | import java.nio.channels.NonWritableChannelException; 20 | 21 | import net.smacke.jaydio.buffer.JavaHeapByteBuffer; 22 | import net.smacke.jaydio.buffer.JaydioByteBuffer; 23 | import net.smacke.jaydio.channel.MockByteChannel; 24 | 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | 28 | /** 29 | * Test class which composes a {@link BufferedChannelAbstractTester}, passing in a 30 | * {@link MockByteChannel} to test. Since this class is using a mock channel, it 31 | * does not do anything with JNA or any actual file I/O and is therefore fast-testable.

32 | * 33 | * @author smacke 34 | * 35 | */ 36 | public class TestMockByteChannel extends BufferedChannelAbstractTester { 37 | private static final int BLOCK_SIZE = 512; 38 | private static final int BUFFER_SIZE = 2*BLOCK_SIZE; 39 | 40 | @Override 41 | @Before public void setUp() { 42 | channel = MockByteChannel.getChannel(BUFFER_SIZE, BLOCK_SIZE, false); 43 | buffer = JavaHeapByteBuffer.allocate(512); 44 | testPosition = BLOCK_SIZE; 45 | } 46 | 47 | @Override 48 | @Test(expected = NonWritableChannelException.class) 49 | public void testCannotWriteToReadOnlyChannel() throws IOException { 50 | super.tearDown(); 51 | //repoen for readonly 52 | channel = MockByteChannel.getChannel(BUFFER_SIZE, BLOCK_SIZE, true); 53 | buffer = JavaHeapByteBuffer.allocate(BLOCK_SIZE); 54 | super.testCannotWriteToReadOnlyChannel(); 55 | } 56 | } 57 | --------------------------------------------------------------------------------