1K
33 | * @param byteSize
34 | * @return string representation of the byte size
35 | */
36 | def toHumanReadableFormat(byteSize:Long) : String = {
37 | // kilo, mega, giga, tera, peta, exa, zetta, yotta
38 | val unitName = Seq("", "K", "M", "G", "T", "P", "E", "Z", "Y")
39 |
40 | def loop(index:Int, v:Long) : (Long, String) = {
41 | if(index >= unitName.length)
42 | (byteSize, "")
43 | val next = v >> 10L
44 | if(next == 0L)
45 | (v, unitName(index))
46 | else
47 | loop(index+1, next)
48 | }
49 |
50 | val (prefix, unit) = if(byteSize > 0)
51 | loop(0, byteSize)
52 | else
53 | loop(0, -byteSize) match { case (p, u) => (-p, u)}
54 | s"$prefix$unit"
55 | }
56 |
57 | // implicit class DataSize(size:Long) {
58 | // def K = size * KB
59 | // def M = size * MB
60 | // def G = size * GB
61 | // def P = size * PB
62 | // }
63 |
64 | // data size unit
65 | val KB = 1L << 10
66 | val MB = 1L << 20
67 | val GB = 1L << 30
68 | val PB = 1L << 40
69 |
70 | }
71 |
--------------------------------------------------------------------------------
/larray-buffer/src/test/scala/xerial/larray/LArraySpec.scala:
--------------------------------------------------------------------------------
1 | package xerial.larray
2 |
3 | import org.scalatest._
4 | import wvlet.log.LogFormatter.SourceCodeLogFormatter
5 | import wvlet.log.{LogSupport, Logger}
6 | import wvlet.log.io.{ResourceReader, StopWatch, Timer}
7 |
8 | import scala.language.implicitConversions
9 |
10 | /**
11 | * @author Taro L. Saito
12 | */
13 | trait LArraySpec
14 | extends WordSpec
15 | with Matchers
16 | with ResourceReader
17 | with Timer
18 | with LogSupport
19 | with BeforeAndAfterAll
20 | with BeforeAndAfter
21 | with BeforeAndAfterEach
22 | with GivenWhenThen {
23 |
24 | implicit def toTag(t:String) = Tag(t)
25 |
26 | Logger.setDefaultFormatter(SourceCodeLogFormatter)
27 |
28 | override protected def beforeAll(): Unit = {
29 | // Run LogLevel scanner (log-test.properties or log.properties in classpath) every 1 minute
30 | Logger.scheduleLogLevelScan
31 | super.beforeAll()
32 | }
33 |
34 | override protected def afterAll(): Unit = {
35 | Logger.stopScheduledLogLevelScan
36 | super.afterAll()
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/larray-buffer/src/test/scala/xerial/larray/buffer/LBufferTest.scala:
--------------------------------------------------------------------------------
1 | //--------------------------------------
2 | //
3 | // RawByteArrayTestTest.scala
4 | // Since: 2013/12/03 12:02 PM
5 | //
6 | //--------------------------------------
7 |
8 | package xerial.larray.buffer
9 |
10 | import java.nio.ByteBuffer
11 |
12 | import xerial.larray.{DataUnit, LArraySpec}
13 |
14 | /**
15 | * @author Taro L. Saito
16 | */
17 | class LBufferTest extends LArraySpec {
18 |
19 | implicit class RichArray(m:LBuffer) {
20 | def toCSV = m.toArray.mkString(", ")
21 | }
22 |
23 |
24 | "LBuffer" should {
25 |
26 | "allocate memory" in {
27 | val size = 1000
28 | val m = new LBuffer(size)
29 | m.putInt(0, 0)
30 | m.putInt(4, 1)
31 | m.putInt(8, 130)
32 |
33 | m.getInt(0) shouldBe 0
34 | m.getInt(4) shouldBe 1
35 | m.getInt(8) shouldBe 130
36 |
37 | m.size() shouldBe size.toLong
38 |
39 | (0 until size).foreach(i => m.putByte(i, (i % 128).toByte))
40 | (0 until size).forall(i => m.getByte(i) == (i % 128).toByte) should be (true)
41 |
42 | m.clear()
43 |
44 | (0 until size).forall(i => m.getByte(i) == 0) should be (true)
45 |
46 | m.release()
47 | }
48 |
49 | "convert to array" in {
50 | val size = 12
51 | val m = new LBuffer(size);
52 | for(i <- 0 until size)
53 | m(i) = i.toByte
54 | debug(m.toCSV)
55 |
56 | m.clear()
57 | debug(m.toCSV)
58 | }
59 |
60 |
61 | "allocate in single-thread" taggedAs("bench-single") in {
62 |
63 | val N = 100
64 | def range = (0 until N)
65 | val R = 2
66 | val S = 1024 * 1024
67 |
68 | info("start buffer allocation test")
69 |
70 | time("single-thread allocation", repeat=10, blockRepeat = R) {
71 | block("without zero-filling") {
72 | for(i <- range) yield {
73 | new LBuffer(S)
74 | }
75 | }
76 |
77 | block("with zero-filling") {
78 | for(i <- range) yield {
79 | val m = new LBuffer(S)
80 | m.clear()
81 | m
82 | }
83 | }
84 |
85 | block("java array") {
86 | for(i <- range) yield {
87 | new Array[Byte](S)
88 | }
89 | }
90 |
91 | block("byte buffer") {
92 | for(i <- range) yield {
93 | ByteBuffer.allocate(S)
94 | }
95 | }
96 |
97 | block("direct byte buffer") {
98 | for(i <- range) yield {
99 | ByteBuffer.allocateDirect(S)
100 | }
101 | }
102 |
103 | }
104 | }
105 |
106 | "allocate concurrently" taggedAs("bench") in {
107 |
108 | val N = 100
109 | def range = (0 until N).par
110 | val R = 2
111 | val S = 1024 * 1024
112 |
113 | info("start buffer allocation test")
114 |
115 | time("concurrent allocation", repeat=10, blockRepeat = R) {
116 | block("without zero-filling") {
117 | for(i <- range) yield {
118 | new LBuffer(S)
119 | }
120 | }
121 |
122 | block("with zero-filling") {
123 | for(i <- range) yield {
124 | val m = new LBuffer(S)
125 | m.clear()
126 | m
127 | }
128 | }
129 |
130 | block("java array") {
131 | for(i <- range) yield {
132 | new Array[Byte](S)
133 | }
134 | }
135 |
136 | block("byte buffer") {
137 | for(i <- range) yield {
138 | ByteBuffer.allocate(S)
139 | }
140 | }
141 |
142 | block("direct byte buffer") {
143 | for(i <- range) yield {
144 | ByteBuffer.allocateDirect(S)
145 | }
146 | }
147 |
148 | }
149 | }
150 |
151 | "Use less memory" taggedAs("heap") in {
152 |
153 | // Need to produce meaningful memory usage
154 | pending
155 |
156 | val N = 100000
157 | val M = 1024
158 |
159 | val rt = Runtime.getRuntime
160 |
161 | case class Report(tag:String, free:Long, offHeap:Long, total:Long) {
162 | override def toString = s"[${tag}] free:${DataUnit.toHumanReadableFormat(free)}, offheap:${DataUnit.toHumanReadableFormat(offHeap)}"
163 | }
164 |
165 | val memUsage = Seq.newBuilder[Report]
166 |
167 | def report(tag:String) = {
168 | val offHeap = LBufferConfig.allocator.allocatedSize()
169 | val rep = Report(tag, rt.freeMemory(), offHeap, rt.totalMemory())
170 | //memUsage += rep
171 | rep
172 | }
173 |
174 | var r1 : Seq[Array[Byte]] = null
175 | var r2 : Seq[LBuffer] = null
176 |
177 | time("memory allocation", repeat=3) {
178 | Thread.sleep(5000)
179 | block("Array") {
180 | info(report("Array"))
181 | val result = for(i <- 0 until N) yield {
182 | val a = new Array[Byte](M)
183 | report("Array")
184 | a
185 | }
186 | info(report("Array"))
187 | }
188 |
189 | // info("gc")
190 | // System.gc()
191 | Thread.sleep(5000)
192 |
193 | block("LBuffer") {
194 | info(report("LBuffer"))
195 | val result = for(i <- 0 until N) yield {
196 | val l = new LBuffer(M)
197 | report("LBuffer")
198 | l
199 | }
200 | info(report("LBuffer"))
201 | }
202 |
203 | }
204 |
205 |
206 | }
207 |
208 | "read from ByteBuffer" in {
209 | val bytes = Array[Byte](1, 2, 3)
210 | val byteBuffer = ByteBuffer.wrap(bytes)
211 | val lbuffer = new LBuffer(3)
212 | lbuffer.readFrom(byteBuffer, 0)
213 | byteBuffer.array() === lbuffer.toArray
214 | }
215 | }
216 | }
--------------------------------------------------------------------------------
/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // MemoryAllocatorTest.scala
19 | // Since: 2013/03/22 12:02
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.buffer
24 |
25 | import xerial.larray.LArraySpec
26 |
27 |
28 | /**
29 | * @author Taro L. Saito
30 | */
31 | class MemoryAllocatorTest extends LArraySpec {
32 | "ConcurrentMemoryAllocator" should {
33 | "perform better than the default heap allocator" in {
34 |
35 | val N = 1000
36 | val B = 64 * 1024
37 |
38 | val t = time("alloc", repeat = 5) {
39 | block("concurrent") {
40 | val l = for (i <- (0 until N).par) yield {
41 | val a = new LBuffer(B)
42 | a(B-1) = 1.toByte
43 | a
44 | }
45 | l.foreach(_.release)
46 | }
47 |
48 | block("Array") {
49 | val l = for (i <- (0 until N).par) yield {
50 | val a = new Array[Int](B)
51 | a(B-1) = 1
52 | a
53 | }
54 | }
55 | }
56 | t("concurrent") should be <= (t("Array"))
57 | }
58 | }
59 | }
--------------------------------------------------------------------------------
/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala:
--------------------------------------------------------------------------------
1 | //--------------------------------------
2 | //
3 | // WrappedLBufferTest.scala
4 | // Since: 2013/12/11 23:07
5 | //
6 | //--------------------------------------
7 |
8 | package xerial.larray.buffer
9 |
10 | import xerial.larray.LArraySpec
11 |
12 | /**
13 | * @author Taro L. Saito
14 | */
15 | class WrappedLBufferTest extends LArraySpec {
16 |
17 | "WrappedLBuffer" should {
18 |
19 | "be a subrange of LBuffer" in {
20 | val l = new LBuffer(10)
21 | for(i <- 0 until l.size().toInt) {
22 | l(i) = (10 - i).toByte
23 | }
24 |
25 | debug(l.toArray.mkString(", "))
26 | val v = l.view(3, 8)
27 |
28 | debug(v.toArray.mkString(", "))
29 | v.size() shouldBe 8 - 3
30 | v.toArray.zipWithIndex.forall{case (a, i) => a == l(i+3)}
31 | }
32 |
33 | }
34 | }
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/impl/LArrayLoader.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayLoader.scala
19 | // Since: 2013/03/18 10:35 AM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.impl;
24 |
25 |
26 | import java.io.*;
27 | import java.security.DigestInputStream;
28 | import java.security.MessageDigest;
29 | import java.security.NoSuchAlgorithmException;
30 | import java.util.UUID;
31 |
32 |
33 | /**
34 | * Internal only - Do not use this class. This class loads a native
35 | * library of larray (larray.dll, liblarray.so, etc.) according to the
36 | * user platform (os.name and os.arch).
37 |
38 | * In default, no configuration is required to use larray, but you can load
39 | * your own native library created by 'make native' command.
40 | *
41 | * This LArrayLoader load a native library (larray.dll, larray.so, etc.) using the following procedure:
42 | *
43 | * - Extract one of the libraries embedded in larray-(version).jar into
44 | * (System property: java.io.tempdir). If
45 | * xerial.larray.tempdir is set, use this folder instead of
46 | * java.io.tempdir.
47 | *
48 | *
49 | *
50 | * If you do not want to use folder java.io.tempdir, set the System
51 | * property xerial.larray.tempdir. For example, to use
52 | * /tmp/leo as a temporary folder to copy native libraries, use -D option
53 | * of JVM:
54 | *
55 | *
56 | *
57 | * java -Dxerial.larray.tempdir="/tmp/leo" ...
58 | *
59 | *
60 | *
61 | *
62 | *
63 | * @author leo
64 | *
65 | */
66 | class LArrayLoader {
67 |
68 | public static String KEY_LARRAY_TEMPDIR = "xerial.larray.tempdir";
69 |
70 | private static boolean isLoaded = false;
71 | private static Object api = null;
72 |
73 | public static void setApi(Object nativeInstance) {
74 | api = nativeInstance;
75 | isLoaded = true;
76 | }
77 |
78 | public static synchronized void load() throws Exception {
79 | if(isLoaded)
80 | return;
81 |
82 | try {
83 | File libFile = findNativeLibrary().newCopy();
84 | // Delete the extracted native library upon exit
85 | libFile.deleteOnExit();
86 | System.load(libFile.getAbsolutePath());
87 | isLoaded = true;
88 | }
89 | catch(Exception e) {
90 | System.err.println(e.getMessage());
91 | throw e;
92 | }
93 | }
94 |
95 | /**
96 | * Computes the MD5 value of the input stream
97 | *
98 | * @param input
99 | * @return
100 | * @throws IOException
101 | * @throws IllegalStateException
102 | */
103 | public static String md5sum(InputStream input) throws IOException {
104 | InputStream in = new BufferedInputStream(input);
105 | try {
106 | MessageDigest digest = java.security.MessageDigest.getInstance("MD5");
107 | DigestInputStream digestInputStream = new DigestInputStream(in, digest);
108 | while(digestInputStream.read() >= 0) {
109 | }
110 | OutputStream md5out = new ByteArrayOutputStream();
111 | md5out.write(digest.digest());
112 | return md5out.toString();
113 | }
114 | catch(NoSuchAlgorithmException e) {
115 | throw new IllegalStateException("MD5 algorithm is not available: " + e.getMessage());
116 | }
117 | finally {
118 | in.close();
119 | }
120 | }
121 |
122 |
123 | static class NativeLib{
124 |
125 | private final String nativeLibFolder;
126 | private final String libName;
127 |
128 | public NativeLib(String nativeLibFolder, String libName) {
129 | this.nativeLibFolder = nativeLibFolder;
130 | this.libName = libName;
131 | }
132 |
133 | /** Create a new unique copy of the native library **/
134 | public File newCopy() throws IOException {
135 | // Temporary library folder. Use the value of xerial.larray.tempdir or java.io.tmpdir
136 | String tempFolder = new File(System.getProperty(KEY_LARRAY_TEMPDIR, System.getProperty("java.io.tmpdir"))).getAbsolutePath();
137 | // Extract and load a native library inside the jar file
138 | return extractLibraryFile(nativeLibFolder, libName, tempFolder);
139 | }
140 |
141 |
142 | private boolean contentsEquals(InputStream in1, InputStream in2) throws IOException {
143 | if(!(in1 instanceof BufferedInputStream)) {
144 | in1 = new BufferedInputStream(in1);
145 | }
146 | if(!(in2 instanceof BufferedInputStream)) {
147 | in2 = new BufferedInputStream(in2);
148 | }
149 |
150 | int ch = in1.read();
151 | while(ch != -1) {
152 | int ch2 = in2.read();
153 | if(ch != ch2)
154 | return false;
155 | ch = in1.read();
156 | }
157 | int ch2 = in2.read();
158 | return ch2 == -1;
159 | }
160 |
161 | /**
162 | * Extract the specified library file to the target folder
163 | *
164 | * @param libFolderForCurrentOS
165 | * @param libraryFileName
166 | * @param targetFolder
167 | * @return
168 | */
169 | private File extractLibraryFile(String libFolderForCurrentOS, String libraryFileName, String targetFolder) throws IOException {
170 | String nativeLibraryFilePath = libFolderForCurrentOS + "/" + libraryFileName;
171 | String suffix = UUID.randomUUID().toString().substring(0, 8);
172 | String extractedLibFileName = String.format("larray-%s.lib", suffix);
173 | File extractedLibFile = new File(targetFolder, extractedLibFileName);
174 |
175 |
176 | // Extract a native library file into the target directory
177 | InputStream reader = this.getClass().getResourceAsStream(nativeLibraryFilePath);
178 | OutputStream writer = new FileOutputStream(extractedLibFile);
179 | try {
180 | byte[] buffer = new byte[8192];
181 | int bytesRead = 0;
182 | while ((bytesRead = reader.read(buffer)) != -1) {
183 | writer.write(buffer, 0, bytesRead);
184 | }
185 | }
186 | finally {
187 | extractedLibFile.deleteOnExit();
188 | if(writer != null)
189 | writer.close();
190 | if(reader != null)
191 | reader.close();
192 | }
193 |
194 | // Set executable (x) flag to enable Java to load the native library
195 | extractedLibFile.setReadable(true);
196 | extractedLibFile.setWritable(true, true);
197 | extractedLibFile.setExecutable(true);
198 |
199 |
200 | // Check whether the contents are properly copied from the resource folder
201 | {
202 | InputStream nativeIn = LArrayLoader.class.getResourceAsStream(nativeLibraryFilePath);
203 | InputStream extractedLibIn = new FileInputStream(extractedLibFile);
204 | try {
205 | if(!contentsEquals(nativeIn, extractedLibIn))
206 | throw new IOException(String.format("Failed to write a native library file at %s", extractedLibFile));
207 | }
208 | finally {
209 | if(nativeIn != null)
210 | nativeIn.close();
211 | if(extractedLibIn != null)
212 | extractedLibIn.close();
213 | }
214 | }
215 |
216 |
217 | return new File(targetFolder, extractedLibFileName);
218 | }
219 |
220 | }
221 |
222 | private static boolean hasResource(String path) {
223 | return LArrayLoader.class.getResource(path) != null;
224 | }
225 |
226 | private static NativeLib findNativeLibrary() {
227 |
228 | // Try to load the library in xerial.larray.native */
229 | // Resolve the library file name with a suffix (e.g., dll, .so, etc.)
230 | String nativeLibraryName = System.mapLibraryName("larray");
231 | // Load an OS-dependent native library inside a jar file
232 | String nativeLibraryPath = "/xerial/larray/native/" + OSInfo.getNativeLibFolderPathForCurrentOS();
233 | boolean hasNativeLib = hasResource(nativeLibraryPath + "/" + nativeLibraryName);
234 |
235 | // Fix for openjdk7 for Mac
236 | if(!hasNativeLib) {
237 | if(OSInfo.getOSName().equals("Mac")) {
238 | String altName = "liblarray.jnilib";
239 | if(hasResource(nativeLibraryPath + "/" + altName)) {
240 | nativeLibraryName = altName;
241 | hasNativeLib = true;
242 | }
243 | }
244 | }
245 |
246 | if(!hasNativeLib) {
247 | String errorMessage = String.format("no native library is found for os.name=%s and os.arch=%s", OSInfo.getOSName(), OSInfo.getArchName());
248 | System.err.println(errorMessage);
249 | }
250 |
251 | return new NativeLib(nativeLibraryPath, nativeLibraryName);
252 | }
253 |
254 |
255 | }
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/impl/LArrayNative.c:
--------------------------------------------------------------------------------
1 | #include "LArrayNative.h"
2 | #include
3 | #include
4 |
5 | #if defined(_WIN32 )|| defined(_WIN64)
6 | #include
7 | #else
8 | #include
9 | #endif
10 |
11 | /*
12 | * Class: xerial_larray_impl_LArrayNative
13 | * Method: copyToArray
14 | * Signature: (JLjava/lang/Object;II)I
15 | */
16 | JNIEXPORT jint JNICALL Java_xerial_larray_impl_LArrayNative_copyToArray
17 | (JNIEnv *env, jclass cls, jlong srcAddr, jobject destArr, jint destOffset, jint len) {
18 |
19 | char* ptr = (char*) (*env)->GetPrimitiveArrayCritical(env, (jarray) destArr, 0);
20 | memcpy(ptr + destOffset, (void*) srcAddr, (size_t) len);
21 | (*env)->ReleasePrimitiveArrayCritical(env, (jarray) destArr, ptr, 0);
22 |
23 | return len;
24 | }
25 |
26 | /*
27 | * Class: xerial_larray_impl_LArrayNative
28 | * Method: copyFromArray
29 | * Signature: (Ljava/lang/Object;IJI)I
30 | */
31 | JNIEXPORT jint JNICALL Java_xerial_larray_impl_LArrayNative_copyFromArray
32 | (JNIEnv *env, jclass cls, jobject srcArr, jint srcOffset, jlong destAddr, jint len) {
33 |
34 | char* ptr = (char*) (*env)->GetPrimitiveArrayCritical(env, (jarray) srcArr, 0);
35 | memcpy((void*) destAddr, (void*) (ptr + srcOffset), (size_t) len);
36 | (*env)->ReleasePrimitiveArrayCritical(env, (jarray) srcArr, ptr, 0);
37 |
38 | return len;
39 | }
40 |
41 |
42 |
43 | JNIEXPORT jlong JNICALL Java_xerial_larray_impl_LArrayNative_mmap
44 | (JNIEnv *env, jclass cls, jlong fd, jint mode, jlong offset, jlong size)
45 | {
46 | #if defined(_WIN32) || defined(_WIN64)
47 | void *mapAddress = 0;
48 | jlong maxSize = offset + size;
49 | jint lowLen = (jint) (maxSize);
50 | jint highLen = (jint) (maxSize >> 32);
51 | jint lowOffset = (jint) offset;
52 | jint highOffset = (jint) (offset >> 32);
53 | HANDLE fileHandle = (HANDLE) fd;
54 | HANDLE mapping;
55 | DWORD mapAccess = FILE_MAP_READ;
56 | DWORD fileProtect = PAGE_READONLY;
57 | BOOL result;
58 | if (mode == 0) {
59 | fileProtect = PAGE_READONLY;
60 | mapAccess = FILE_MAP_READ;
61 | } else if (mode == 1) {
62 | fileProtect = PAGE_READWRITE;
63 | mapAccess = FILE_MAP_WRITE;
64 | } else if (mode == 2) {
65 | fileProtect = PAGE_WRITECOPY;
66 | mapAccess = FILE_MAP_COPY;
67 | }
68 |
69 | mapping = CreateFileMapping(fileHandle, NULL, fileProtect, highLen, lowLen, NULL);
70 | mapAddress = MapViewOfFile(mapping, mapAccess, highOffset, lowOffset, (size_t) size);
71 |
72 | result = CloseHandle(mapping);
73 | return (jlong) mapAddress;
74 |
75 | #else
76 | void *addr;
77 | int prot = 0;
78 | int flags = 0;
79 | if(mode == 0) {
80 | prot = PROT_READ;
81 | flags = MAP_SHARED;
82 | } else if(mode == 1) {
83 | prot = PROT_READ | PROT_WRITE;
84 | flags = MAP_SHARED;
85 | } else if(mode == 2) {
86 | prot = PROT_READ | PROT_WRITE;
87 | flags = MAP_PRIVATE;
88 | }
89 |
90 | addr = mmap(0, size, prot, flags, (int) fd, offset);
91 |
92 | return (jlong) addr;
93 | #endif
94 | }
95 |
96 |
97 |
98 | JNIEXPORT void JNICALL Java_xerial_larray_impl_LArrayNative_munmap
99 | (JNIEnv *env, jclass cls, jlong addr, jlong size) {
100 | #if defined(_WIN32) || defined(_WIN64)
101 | void *a = (void *) addr;
102 | BOOL result;
103 | result = UnmapViewOfFile(a);
104 | #else
105 | munmap((void *) addr, (size_t) size);
106 | #endif
107 |
108 | }
109 |
110 |
111 | JNIEXPORT void JNICALL Java_xerial_larray_impl_LArrayNative_msync
112 | (JNIEnv *env, jclass cls, jlong fd, jlong addr, jlong size) {
113 |
114 | #if defined(_WIN32) || defined(_WIN64)
115 | void *a = (void *) addr;
116 | BOOL result;
117 | int retry;
118 |
119 | /*
120 | * FlushViewOfFile can fail with ERROR_LOCK_VIOLATION if the memory
121 | * system is writing dirty pages to disk. As there is no way to
122 | * synchronize the flushing then we retry a limited number of times.
123 | */
124 | retry = 0;
125 | do {
126 | result = FlushViewOfFile(a, (DWORD)size);
127 | if ((result != 0) || (GetLastError() != ERROR_LOCK_VIOLATION))
128 | break;
129 | retry++;
130 | } while (retry < 3);
131 |
132 | if(fd != 0 && result != 0) {
133 | result = FlushFileBuffers((HANDLE)fd);
134 | if(result == 0 && GetLastError() == ERROR_ACCESS_DENIED) {
135 | // read-only mapping
136 | result = 1;
137 | }
138 | }
139 | if(result == 0)
140 | printf("failed to sync\n");
141 |
142 | #else
143 | msync((void *) addr, (size_t) size, MS_SYNC);
144 | #endif
145 | }
146 |
147 | JNIEXPORT jlong JNICALL Java_xerial_larray_impl_LArrayNative_duplicateHandle
148 | (JNIEnv *env, jclass cls, jlong handle) {
149 |
150 | #if defined(_WIN32) || defined(_WIN64)
151 | HANDLE hProcess = GetCurrentProcess();
152 | HANDLE hFile = (HANDLE) handle;
153 | HANDLE hResult;
154 | BOOL res = DuplicateHandle(hProcess, hFile, hProcess, &hResult, 0, FALSE, DUPLICATE_SAME_ACCESS);
155 | if(res == 0)
156 | return -1;
157 | else
158 | return (jlong) hResult;
159 | #else
160 | return -1;
161 | #endif
162 |
163 | }
164 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/impl/LArrayNative.h:
--------------------------------------------------------------------------------
1 | /* DO NOT EDIT THIS FILE - it is machine generated */
2 | #include
3 | /* Header for class xerial_larray_impl_LArrayNative */
4 |
5 | #ifndef _Included_xerial_larray_impl_LArrayNative
6 | #define _Included_xerial_larray_impl_LArrayNative
7 | #ifdef __cplusplus
8 | extern "C" {
9 | #endif
10 | /*
11 | * Class: xerial_larray_impl_LArrayNative
12 | * Method: copyToArray
13 | * Signature: (JLjava/lang/Object;II)I
14 | */
15 | JNIEXPORT jint JNICALL Java_xerial_larray_impl_LArrayNative_copyToArray
16 | (JNIEnv *, jclass, jlong, jobject, jint, jint);
17 |
18 | /*
19 | * Class: xerial_larray_impl_LArrayNative
20 | * Method: copyFromArray
21 | * Signature: (Ljava/lang/Object;IJI)I
22 | */
23 | JNIEXPORT jint JNICALL Java_xerial_larray_impl_LArrayNative_copyFromArray
24 | (JNIEnv *, jclass, jobject, jint, jlong, jint);
25 |
26 | /*
27 | * Class: xerial_larray_impl_LArrayNative
28 | * Method: mmap
29 | * Signature: (JIJJ)J
30 | */
31 | JNIEXPORT jlong JNICALL Java_xerial_larray_impl_LArrayNative_mmap
32 | (JNIEnv *, jclass, jlong, jint, jlong, jlong);
33 |
34 | /*
35 | * Class: xerial_larray_impl_LArrayNative
36 | * Method: munmap
37 | * Signature: (JJ)V
38 | */
39 | JNIEXPORT void JNICALL Java_xerial_larray_impl_LArrayNative_munmap
40 | (JNIEnv *, jclass, jlong, jlong);
41 |
42 | /*
43 | * Class: xerial_larray_impl_LArrayNative
44 | * Method: msync
45 | * Signature: (JJJ)V
46 | */
47 | JNIEXPORT void JNICALL Java_xerial_larray_impl_LArrayNative_msync
48 | (JNIEnv *, jclass, jlong, jlong, jlong);
49 |
50 | /*
51 | * Class: xerial_larray_impl_LArrayNative
52 | * Method: duplicateHandle
53 | * Signature: (J)J
54 | */
55 | JNIEXPORT jlong JNICALL Java_xerial_larray_impl_LArrayNative_duplicateHandle
56 | (JNIEnv *, jclass, jlong);
57 |
58 | #ifdef __cplusplus
59 | }
60 | #endif
61 | #endif
62 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/impl/LArrayNative.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.impl;
17 |
18 | /**
19 | * LArray native code interface
20 | * @author Taro L. Saito
21 | */
22 | public class LArrayNative {
23 |
24 | static {
25 | try {
26 | LArrayLoader.load();
27 | }
28 | catch (Exception e) {
29 | e.printStackTrace();
30 | }
31 | }
32 |
33 | public static native int copyToArray(long srcAddress, Object destArray, int destOffset, int length);
34 | public static native int copyFromArray(Object srcArray, int srcOffset, long destAddress, int length);
35 |
36 | public static native long mmap(long fd, int mode, long offset, long size);
37 | public static native void munmap(long address, long size);
38 | public static native void msync(long handle, long address, long size);
39 | public static native long duplicateHandle(long handle);
40 | }
41 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/impl/OSInfo.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2008 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | // snappy-java Project
18 | //
19 | // OSInfo.java
20 | // Since: May 20, 2008
21 | //
22 | // $URL$
23 | // $Author$
24 | //--------------------------------------
25 | package xerial.larray.impl;
26 |
27 | import java.io.IOException;
28 |
29 | /**
30 | * Provides OS name and architecture name.
31 | *
32 | * @author leo
33 | */
34 | public class OSInfo {
35 | public static void main(String[] args) {
36 | if (args.length >= 1) {
37 | if ("--os".equals(args[0])) {
38 | System.out.print(getOSName());
39 | return;
40 | } else if ("--arch".equals(args[0])) {
41 | System.out.print(getArchName());
42 | return;
43 | }
44 | }
45 |
46 | System.out.print(getNativeLibFolderPathForCurrentOS());
47 | }
48 |
49 | public static String getNativeLibFolderPathForCurrentOS() {
50 | return getOSName() + "/" + getArchName();
51 | }
52 |
53 | public static String getOSName() {
54 | return translateOSNameToFolderName(System.getProperty("os.name"));
55 | }
56 |
57 |
58 | public static String getArchName() {
59 | // if running Linux on ARM, need to determine ABI of JVM
60 | String osArch = System.getProperty("os.arch");
61 | if (osArch.startsWith("arm") && System.getProperty("os.name").contains("Linux")) {
62 | String javaHome = System.getProperty("java.home");
63 | try {
64 | // determine if first JVM found uses ARM hard-float ABI
65 | String[] cmdarray = {"/bin/sh", "-c", "find '" + javaHome +
66 | "' -name 'libjvm.so' | head -1 | xargs readelf -A | " +
67 | "grep 'Tag_ABI_VFP_args: VFP registers'"};
68 | int exitCode = Runtime.getRuntime().exec(cmdarray).waitFor();
69 | if (exitCode == 0)
70 | return "armhf";
71 | } catch (IOException e) {
72 | // ignored: fall back to "arm" arch (soft-float ABI)
73 | } catch (InterruptedException e) {
74 | // ignored: fall back to "arm" arch (soft-float ABI)
75 | }
76 | } else if (getOSName().equals("Mac") && (osArch.equals("universal") || osArch.equals("amd64"))) {
77 | return "x86_64"; // Fix for openjdk7
78 | }
79 | return translateArchNameToFolderName(osArch);
80 | }
81 |
82 | static String translateOSNameToFolderName(String osName) {
83 | if (osName.contains("Windows")) {
84 | return "Windows";
85 | } else if (osName.contains("Mac")) {
86 | return "Mac";
87 | } else if (osName.contains("Linux")) {
88 | return "Linux";
89 | } else {
90 | return osName.replaceAll("\\W", "");
91 | }
92 | }
93 |
94 | public static boolean isWindows() {
95 | return getOSName().contains("Windows");
96 | }
97 |
98 |
99 | static String translateArchNameToFolderName(String archName) {
100 | return archName.replaceAll("\\W", "");
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java:
--------------------------------------------------------------------------------
1 | package xerial.larray.mmap;
2 |
3 |
4 | import java.io.File;
5 | import java.io.FileDescriptor;
6 | import java.io.IOException;
7 | import java.io.RandomAccessFile;
8 | import java.lang.reflect.Field;
9 | import java.nio.channels.FileChannel;
10 | import sun.misc.SharedSecrets;
11 | import xerial.larray.buffer.LBufferAPI;
12 | import xerial.larray.buffer.LBufferConfig;
13 | import xerial.larray.buffer.UnsafeUtil;
14 | import xerial.larray.impl.LArrayNative;
15 | import xerial.larray.impl.OSInfo;
16 |
17 | /**
18 | * Memory-mapped file buffer
19 | *
20 | * @author Taro L. Saito
21 | */
22 | public class MMapBuffer extends LBufferAPI {
23 |
24 | private final RandomAccessFile raf;
25 | private final FileChannel fc;
26 | private final long fd;
27 | private final int pagePosition;
28 |
29 | private final long address;
30 | private long winHandle = -1;
31 |
32 | @Override
33 | public long address() { return address; }
34 |
35 | /**
36 | * Open an memory mapped file.
37 | * @param f
38 | * @param mode
39 | * @throws IOException
40 | */
41 | public MMapBuffer(File f, MMapMode mode) throws IOException {
42 | this(f, 0L, f.length(), mode);
43 | }
44 |
45 | /**
46 | * Open an memory mapped file.
47 | * @param f
48 | * @param offset
49 | * @param size
50 | * @param mode
51 | * @throws IOException
52 | */
53 | public MMapBuffer(File f, long offset, long size, MMapMode mode) throws IOException {
54 | super();
55 | this.raf = new RandomAccessFile(f, mode.mode);
56 | this.fc = raf.getChannel();
57 | // Retrieve file descriptor
58 | FileDescriptor rawfd = raf.getFD();
59 | try {
60 | if(!OSInfo.isWindows()) {
61 | Field idf = rawfd.getClass().getDeclaredField("fd");
62 | idf.setAccessible(true);
63 | this.fd = idf.getInt(rawfd);
64 | }
65 | else {
66 | // In Windows, fd is stored as 'handle'
67 | Field idf = rawfd.getClass().getDeclaredField("handle");
68 | idf.setAccessible(true);
69 | this.fd = idf.getLong(rawfd);
70 | }
71 | }
72 | catch(Exception e) {
73 | throw new IOException("Failed to retrieve file descriptor of " + f.getPath() + ": " + e.getMessage());
74 | }
75 |
76 | long allocationGranule = UnsafeUtil.unsafe.pageSize();
77 | this.pagePosition = (int) (offset % allocationGranule);
78 |
79 | // Compute mmap address
80 | if(!fc.isOpen())
81 | throw new IOException("closed " + f.getPath());
82 |
83 | long fileSize = fc.size();
84 | if(fileSize < offset + size) {
85 | // If file size is smaller than the specified size, extend the file size
86 | raf.seek(offset + size - 1);
87 | raf.write(0);
88 | //logger.trace(s"extend file size to ${fc.size}")
89 | }
90 | long mapPosition = offset - pagePosition;
91 | long mapSize = size + pagePosition;
92 | // A workaround for the error when calling fc.map(MapMode.READ_WRITE, offset, size) with size more than 2GB
93 |
94 | long rawAddr = LArrayNative.mmap(fd, mode.code, mapPosition, mapSize);
95 | //trace(f"mmap addr:$rawAddr%x, start address:${rawAddr+pagePosition}%x")
96 |
97 | if(OSInfo.isWindows()) {
98 | sun.misc.JavaIOFileDescriptorAccess a = SharedSecrets.getJavaIOFileDescriptorAccess();
99 | winHandle = LArrayNative.duplicateHandle(a.getHandle(raf.getFD()));
100 | //debug(f"win handle: $winHandle%x")
101 | }
102 |
103 | this.m = new MMapMemory(rawAddr, mapSize);
104 | LBufferConfig.allocator.register(m);
105 |
106 | this.address = rawAddr + pagePosition;
107 | }
108 |
109 | /**
110 | * Forces any changes made to this buffer to be written to the file
111 | */
112 | public void flush() {
113 | LArrayNative.msync(winHandle, m.headerAddress(), m.size());
114 | }
115 |
116 | /**
117 | * Close the memory mapped file. To ensure the written data is saved in the file, call flush before closing.
118 | */
119 | public void close() throws IOException {
120 | release();
121 | fc.close();
122 | }
123 |
124 | protected long offset() {
125 | return pagePosition;
126 | }
127 |
128 | }
129 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/mmap/MMapMemory.java:
--------------------------------------------------------------------------------
1 | package xerial.larray.mmap;
2 |
3 | import xerial.larray.buffer.Memory;
4 | import xerial.larray.impl.LArrayNative;
5 |
6 | import java.lang.ref.ReferenceQueue;
7 |
8 | /**
9 | * @author Taro L. Saito
10 | */
11 | public class MMapMemory implements Memory {
12 | public long address;
13 | public long size;
14 |
15 | public MMapMemory(long address, long size) {
16 | this.address = address;
17 | this.size = size;
18 | }
19 |
20 | public long address() {
21 | return address;
22 | }
23 |
24 | public long size() {
25 | return size;
26 | }
27 |
28 | public long headerAddress() { return address; }
29 |
30 | public MMapMemoryReference toRef(ReferenceQueue queue) {
31 | return new MMapMemoryReference(this, queue);
32 | }
33 |
34 | public long dataSize() { return size; }
35 |
36 | public void release() {
37 | LArrayNative.munmap(address, size);
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/mmap/MMapMemoryReference.java:
--------------------------------------------------------------------------------
1 | package xerial.larray.mmap;
2 |
3 | import xerial.larray.buffer.Memory;
4 | import xerial.larray.buffer.MemoryReference;
5 |
6 | import java.lang.ref.ReferenceQueue;
7 |
8 | /**
9 | * @author Taro L. Saito
10 | */
11 | public class MMapMemoryReference extends MemoryReference {
12 |
13 | public final long size;
14 |
15 | public MMapMemoryReference(Memory m, ReferenceQueue queue) {
16 | super(m, queue);
17 | this.size = m.size();
18 | }
19 |
20 | public String name() { return "mmap"; }
21 |
22 | public MMapMemory toMemory() {
23 | return new MMapMemory(address, size);
24 | }
25 |
26 | }
27 |
28 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/java/xerial/larray/mmap/MMapMode.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.mmap;
17 |
18 | /**
19 | * Modes allowed in memory mapped file
20 | * @author Taro L. Saito
21 | */
22 | public enum MMapMode {
23 |
24 | READ_ONLY(0, "r"),
25 | READ_WRITE(1, "rw"),
26 | PRIVATE(2, "rw");
27 |
28 | public final int code;
29 | public final String mode;
30 |
31 | private MMapMode(int code, String fileOpenMode) {
32 | this.code = code;
33 | this.mode = fileOpenMode;
34 | }
35 |
36 | }
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/VERSION:
--------------------------------------------------------------------------------
1 | version=0.1
2 |
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Linux/amd64/liblarray.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Linux/amd64/liblarray.so
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Linux/arm/liblarray.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Linux/arm/liblarray.so
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Linux/armhf/liblarray.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Linux/armhf/liblarray.so
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Linux/i386/liblarray.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Linux/i386/liblarray.so
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Mac/x86_64/liblarray.jnilib:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Mac/x86_64/liblarray.jnilib
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Windows/amd64/larray.dll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Windows/amd64/larray.dll
--------------------------------------------------------------------------------
/larray-mmap/src/main/resources/xerial/larray/native/Windows/x86/larray.dll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xerial/larray/400d23241f5050dc06f724b4fee7a485b1688bf9/larray-mmap/src/main/resources/xerial/larray/native/Windows/x86/larray.dll
--------------------------------------------------------------------------------
/larray-mmap/src/test/scala/xerial/larray/impl/LArrayLoaderTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayLoaderTest.scala
19 | // Since: 2013/03/29 9:56
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.impl
24 |
25 | import java.net.{URL, URLClassLoader}
26 | import java.io.File
27 | import xerial.larray.LArraySpec
28 |
29 |
30 | /**
31 | * @author Taro L. Saito
32 | */
33 | class LArrayLoaderTest extends LArraySpec {
34 |
35 | "LArrayLoader" should {
36 |
37 | "use a different library name for each class loader" in {
38 |
39 | val larrayNativeCls = "xerial.larray.impl.LArrayNative"
40 |
41 | val cl = Thread.currentThread().getContextClassLoader
42 | val parentCl = cl.getParent.getParent // Fix for sbt-0.13
43 | debug(s"context cl: ${cl}, parent cl: $parentCl")
44 | val classPath = Array(new File("larray-mmap/target/classes").toURI.toURL)
45 | val cl1 = new URLClassLoader(classPath, parentCl)
46 | val cl2 = new URLClassLoader(classPath, parentCl)
47 |
48 | import java.{lang=>jl}
49 |
50 | val nativeCls1 = cl1.loadClass(larrayNativeCls)
51 | val ni1 = nativeCls1.newInstance()
52 | val arr1 = Array.ofDim[Byte](100)
53 | val m1 = nativeCls1.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE)
54 | m1.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr1, new jl.Integer(0), new jl.Integer(0)):_*)
55 | val nativeCls1_2 = cl1.loadClass(larrayNativeCls)
56 |
57 | val nativeCls2 = cl2.loadClass(larrayNativeCls)
58 | val ni2 = nativeCls2.newInstance()
59 | val arr2 = Array.ofDim[Byte](100)
60 | val m2 = nativeCls2.getDeclaredMethod("copyToArray", jl.Long.TYPE, classOf[AnyRef], jl.Integer.TYPE, jl.Integer.TYPE)
61 | m2.invoke(ni1, Seq.apply[AnyRef](new jl.Long(0L), arr2, new jl.Integer(0), new jl.Integer(0)):_*)
62 |
63 | nativeCls1 should not be (nativeCls2)
64 | nativeCls1 should be (nativeCls1_2)
65 |
66 | val arr3 = Array.ofDim[Byte](100)
67 | LArrayNative.copyToArray(0, arr3, 0, 0)
68 | }
69 |
70 |
71 | }
72 |
73 | }
--------------------------------------------------------------------------------
/larray/src/main/java/xerial/larray/japi/LArrayJ.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.japi;
17 |
18 | import scala.reflect.ClassTag$;
19 | import xerial.larray.*;
20 | import xerial.larray.buffer.LBufferConfig;
21 | import xerial.larray.buffer.MemoryAllocator;
22 | import xerial.larray.mmap.MMapMode;
23 |
24 | import java.io.File;
25 |
26 | /**
27 | * Java interface of LArray
28 | * @author Taro L. Saito
29 | */
30 | public class LArrayJ {
31 |
32 | static MemoryAllocator defaultAllocator() { return LBufferConfig.allocator; }
33 |
34 | public static MappedLByteArray mmap(File f, MMapMode mode) {
35 | return new MappedLByteArray(f, 0L, f.length(), mode, defaultAllocator());
36 | }
37 |
38 | public static MappedLByteArray mmap(File f, long offset, long size, MMapMode mode) {
39 | return new MappedLByteArray(f, offset, size, mode, defaultAllocator());
40 | }
41 |
42 |
43 | public static LByteArray newLByteArray(long size) {
44 | return new LByteArray(size, defaultAllocator());
45 | }
46 |
47 | public static LCharArray newLCharArray(long size) {
48 | return new LCharArray(size, defaultAllocator());
49 | }
50 |
51 | public static LShortArray newLShortArray(long size) {
52 | return new LShortArray(size, defaultAllocator());
53 | }
54 |
55 | public static LIntArray newLIntArray(long size) {
56 | return new LIntArray(size, defaultAllocator());
57 | }
58 |
59 | public static LFloatArray newLFloatArray(long size) {
60 | return new LFloatArray(size, defaultAllocator());
61 | }
62 |
63 | public static LDoubleArray newLDoubleArray(long size) {
64 | return new LDoubleArray(size, defaultAllocator());
65 | }
66 |
67 | public static LLongArray newLLongArray(long size) {
68 | return new LLongArray(size, defaultAllocator());
69 | }
70 |
71 | public static LBitArray newLBitArray(long size) {
72 | return new LBitArray(size);
73 | }
74 |
75 | public static LIntArray loadLIntArrayFrom(File file) {
76 | return (LIntArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Int());
77 | }
78 |
79 | public static LByteArray loadLByteArrayFrom(File file) {
80 | return (LByteArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Byte());
81 | }
82 |
83 | public static LShortArray loadLShortArrayFrom(File file) {
84 | return (LShortArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Short());
85 | }
86 |
87 | public static LCharArray loadLCharArrayFrom(File file) {
88 | return (LCharArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Char());
89 | }
90 |
91 | public static LFloatArray loadLFloatArrayFrom(File file) {
92 | return (LFloatArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Float());
93 | }
94 |
95 | public static LDoubleArray loadLDoubleArrayFrom(File file) {
96 | return (LDoubleArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Double());
97 | }
98 |
99 | public static LLongArray loadLLongArrayFrom(File file) {
100 | return (LLongArray) LArray$.MODULE$.loadFrom(file, ClassTag$.MODULE$.Long());
101 | }
102 |
103 | }
104 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/AltLArray.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // AltLArray.scala
19 | // Since: 2013/03/25 16:02
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | /**
26 | * A common trait for alternative implementations of LArray. This implementation is provided only for testing purpose, so many features might be missing
27 | * in LArrays impemented this trait.
28 | */
29 | trait AltLIntArrayImpl extends LArray[Int] {
30 |
31 | def address = LArray.EmptyArray.address // throws an exception
32 |
33 | def copyTo(dest:LByteArray, destOffset:Long) {
34 | throw new UnsupportedOperationException("copyTo")
35 | }
36 |
37 | def copyTo[B](srcOffset:Long, dest:RawByteArray[B], destOffset:Long, blen:Long) {
38 | throw new UnsupportedOperationException("copyTo")
39 | }
40 |
41 | def view(from: Long, to: Long) = new LArrayView.LIntArrayView(this, from, to - from)
42 | }
43 |
44 | /**
45 | * Alternative implementation of LArray that might be inefficient, but written for comparing performances.
46 | * LIntArraySimple wraps Array[Int] to support Long-type indexes
47 | * @param size array size
48 | */
49 | class LIntArraySimple(val size: Long) extends LArray[Int] with AltLIntArrayImpl {
50 |
51 | protected[this] def newBuilder = LArray.newBuilder[Int]
52 |
53 |
54 | private def boundaryCheck(i: Long) {
55 | if (i > Int.MaxValue)
56 | sys.error(f"index must be smaller than ${Int.MaxValue}%,d")
57 | }
58 |
59 | private val arr = {
60 | new Array[Int](size.toInt)
61 | }
62 |
63 | def clear() {
64 | java.util.Arrays.fill(arr, 0, size.toInt, 0)
65 | }
66 |
67 | def apply(i: Long): Int = {
68 | //boundaryCheck(i)
69 | arr.apply(i.toInt)
70 | }
71 |
72 | // a(i) = a(j) = 1
73 | def update(i: Long, v: Int): Int = {
74 | //boundaryCheck(i)
75 | arr.update(i.toInt, v)
76 | v
77 | }
78 |
79 | def free {
80 | // do nothing
81 | }
82 |
83 | /**
84 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
85 | */
86 | private[larray] def elementByteSize: Int = 4
87 |
88 |
89 |
90 | }
91 |
92 |
93 | /**
94 | * Emulate large arrays using two-diemensional matrix of Int. Array[Int](page index)(offset in page)
95 | * @param size array size
96 | */
97 | class MatrixBasedLIntArray(val size:Long) extends LArray[Int] with AltLIntArrayImpl {
98 |
99 | private[larray] def elementByteSize: Int = 4
100 |
101 | protected[this] def newBuilder = LArray.newBuilder[Int]
102 |
103 |
104 | private val maskLen : Int = 24
105 | private val B : Int = 1 << maskLen // block size
106 | private val mask : Long = ~(~0L << maskLen)
107 |
108 | @inline private def index(i:Long) : Int = (i >>> maskLen).toInt
109 | @inline private def offset(i:Long) : Int = (i & mask).toInt
110 |
111 | private val numBlocks = ((size + (B - 1L))/ B).toInt
112 | private val arr = Array.ofDim[Int](numBlocks, B)
113 |
114 | def clear() {
115 | for(a <- arr) {
116 | java.util.Arrays.fill(a, 0, a.length, 0)
117 | }
118 | }
119 |
120 | /**
121 | * Retrieve an element
122 | * @param i index
123 | * @return the element value
124 | */
125 | def apply(i: Long) = arr(index(i))(offset(i))
126 |
127 | /**
128 | * Update an element
129 | * @param i index to be updated
130 | * @param v value to set
131 | * @return the value
132 | */
133 | def update(i: Long, v: Int) = {
134 | arr(index(i))(offset(i)) = v
135 | v
136 | }
137 |
138 | /**
139 | * Release the memory of LArray. After calling this method, the results of calling the other methods becomes undefined or might cause JVM crash.
140 | */
141 | def free {}
142 |
143 |
144 | }
145 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LArray2D.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArray2D.scala
19 | // Since: 2013/03/26 13:58
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import reflect.ClassTag
26 |
27 | object LArray2D {
28 |
29 | /**
30 | * Create a new 2D array
31 | * @param rowSize the row size
32 | * @param colSize the column size
33 | * @tparam A the element type
34 | * @return a new 2D array
35 | */
36 | def of[A : ClassTag](rowSize:Long, colSize:Long) = new LArray2D[A](rowSize, colSize)
37 |
38 | }
39 |
40 | /**
41 | * LArray2D is a wrapper of LArray to emulate 2-dimensional array using a single array.
42 | * @author Taro L. Saito
43 | */
44 | class LArray2D[A : ClassTag](val rowSize:Long, val colSize:Long) {
45 | private val arr : LArray[A] = LArray.of[A](rowSize * colSize)
46 |
47 | @inline def pos(i:Long, j:Long) = i * rowSize + j
48 |
49 | def apply(i:Long, j:Long) : A = arr(pos(i, j))
50 | def update(i:Long, j:Long, v:A) : A = arr.update(pos(i,j), v)
51 |
52 | def free { arr.free }
53 |
54 | /**
55 | * Get a view of a row
56 | * @param row the row index
57 | * @return LArrayView[A] of the specified row
58 | */
59 | def row(row:Long) = {
60 | val p = pos(row, 0)
61 | arr.view(p, p+colSize)
62 | }
63 |
64 | /**
65 | * Clear all of the elements in the array
66 | */
67 | def clear() = arr.clear()
68 |
69 | /**
70 | * Get a raw LArray representation of this 2D array
71 | * @return
72 | */
73 | def rawArray : LArray[A] = arr
74 |
75 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LArrayInputStream.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayInputStream.scala
19 | // Since: 2013/03/21 5:28 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.InputStream
26 |
27 | import wvlet.log.LogSupport
28 |
29 | object LArrayInputStream {
30 |
31 | /**
32 | * Create a new InputStream from a given LArray
33 | *
34 | * @param array input
35 | * @tparam A element type
36 | * @return input stream
37 | */
38 | def apply[A](array: LArray[A]): InputStream = {
39 | array match {
40 | case r: RawByteArray[A] => new RawLArrayInputStream[A](r)
41 | case _ => sys.error(s"cannot create InputStream from this LArray class:${array.getClass}")
42 | }
43 | }
44 |
45 | }
46 |
47 | /**
48 | * InputStream implementation for LArrays that uses RawByteArray internally.
49 | *
50 | * @author Taro L. Saito
51 | */
52 | private[larray] class RawLArrayInputStream[A](array: RawByteArray[A]) extends InputStream with LogSupport {
53 |
54 | private var cursor = 0L
55 | private var mark = 0L
56 |
57 | def read() = {
58 | val v = array.getByte(cursor)
59 | cursor += 1
60 | v
61 | }
62 |
63 | override def read(b: Array[Byte], offset: Int, len: Int): Int = {
64 | if (cursor >= array.size) {
65 | -1
66 | }
67 | else {
68 | val readLen = math.min(len, array.byteLength - cursor).toInt
69 | array.writeToArray(cursor, b, offset, readLen)
70 | cursor += readLen
71 | readLen
72 | }
73 | }
74 |
75 | override def available = {
76 | val remaining = array.size - cursor
77 | math.min(Integer.MAX_VALUE, remaining).toInt
78 | }
79 |
80 | override def mark(readlimit: Int) {
81 | // read limit can be ignored since all data is in memory
82 | mark = cursor
83 | }
84 |
85 | override def reset() {
86 | cursor = mark
87 | }
88 |
89 | override def markSupported() = {
90 | true
91 | }
92 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LArrayOutputStream.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayOutputStream.scala
19 | // Since: 2013/03/21 5:54 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.OutputStream
26 | import reflect.ClassTag
27 |
28 | /**
29 | * Create LArray using `java.io.OutputStream` interface
30 | *
31 | * @author Taro L. Saito
32 | */
33 | class LArrayOutputStream[A : ClassTag] extends OutputStream {
34 |
35 | private val buf = new LByteArrayBuilder
36 |
37 | def write(v: Int) {
38 | buf += v.toByte
39 | }
40 |
41 | override def write(b: Array[Byte], off: Int, len: Int) {
42 | buf.append(b, off, len)
43 | }
44 |
45 | def result : LArray[A] = {
46 | val arr = buf.result.asInstanceOf[LByteArray]
47 | LArray.wrap[A](arr.size, arr.m)
48 | }
49 |
50 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LArrayView.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayView.scala
19 | // Since: 2013/03/25 18:04
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import reflect.ClassTag
26 | import xerial.larray.LArray.EmptyArray
27 |
28 | /**
29 | * Provides view of LArray
30 | * @author Taro L. Saito
31 | */
32 | object LArrayView {
33 |
34 | class LBitArrayView(base:LBitArray, offset:Long, val size:Long) extends LArrayView[Boolean] with LBitArrayOps {
35 | protected[this] def newBuilder = new LBitArrayBuilder
36 | def apply(i:Long) = base.apply(i + offset)
37 |
38 | def address = LArray.EmptyArray.address
39 |
40 | /**
41 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
42 | */
43 | private[larray] def elementByteSize = throw new UnsupportedOperationException("elementByteSize of LBitArray")
44 |
45 | /**
46 | * Copy the contents of this LSeq[A] into the target LByteArray
47 | * @param dst
48 | * @param dstOffset
49 | */
50 | def copyTo(dst: LByteArray, dstOffset: Long) {
51 | throw new UnsupportedOperationException("copyTo of LBitArray")
52 | }
53 |
54 | /**
55 | * Copy the contents of this sequence into the target LByteArray
56 | * @param srcOffset
57 | * @param dst
58 | * @param dstOffset
59 | * @param blen the byte length to copy
60 | */
61 | def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) {
62 | throw new UnsupportedOperationException("copyTo of LBitArray")
63 | }
64 |
65 | /**
66 | * Count the number of bits within the specified range [start, end)
67 | * @param checkTrue count true or false
68 | * @param start
69 | * @param end
70 | * @return the number of occurrences
71 | */
72 | def count(checkTrue: Boolean, start: Long, end: Long) = base.count(checkTrue, start+offset, end+offset)
73 |
74 | override def slice(from: Long, until: Long) = base.slice(from+offset, until+offset)
75 | }
76 |
77 | class LByteArrayView(base:LArray[Byte], offset:Long, size:Long) extends AbstractLArrayView[Byte](base, offset, size) {
78 | protected[this] def newBuilder: LBuilder[Byte, LArray[Byte]] = new LByteArrayBuilder
79 | private[larray] def elementByteSize: Int = 1
80 | }
81 |
82 | class LCharArrayView(base:LArray[Char], offset:Long, size:Long) extends AbstractLArrayView[Char](base, offset, size) {
83 | protected[this] def newBuilder: LBuilder[Char, LArray[Char]] = new LCharArrayBuilder
84 | private[larray] def elementByteSize: Int = 2
85 | }
86 |
87 | class LShortArrayView(base:LArray[Short], offset:Long, size:Long) extends AbstractLArrayView[Short](base, offset, size) {
88 | protected[this] def newBuilder: LBuilder[Short, LArray[Short]] = new LShortArrayBuilder
89 | private[larray] def elementByteSize: Int = 2
90 | }
91 |
92 | class LIntArrayView(base:LArray[Int], offset:Long, size:Long) extends AbstractLArrayView[Int](base, offset, size) {
93 | protected[this] def newBuilder: LBuilder[Int, LArray[Int]] = new LIntArrayBuilder
94 | private[larray] def elementByteSize: Int = 4
95 | }
96 |
97 | class LFloatArrayView(base:LArray[Float], offset:Long, size:Long) extends AbstractLArrayView[Float](base, offset, size) {
98 | protected[this] def newBuilder: LBuilder[Float, LArray[Float]] = new LFloatArrayBuilder
99 | private[larray] def elementByteSize: Int = 4
100 | }
101 |
102 | class LLongArrayView(base:LArray[Long], offset:Long, size:Long) extends AbstractLArrayView[Long](base, offset, size) {
103 | protected[this] def newBuilder: LBuilder[Long, LArray[Long]] = new LLongArrayBuilder
104 | private[larray] def elementByteSize: Int = 8
105 | }
106 |
107 | class LDoubleArrayView(base:LArray[Double], offset:Long, size:Long) extends AbstractLArrayView[Double](base, offset, size) {
108 | protected[this] def newBuilder: LBuilder[Double, LArray[Double]] = new LDoubleArrayBuilder
109 | private[larray] def elementByteSize: Int = 8
110 | }
111 |
112 | class LObjectArrayView[A : ClassTag](base:LArray[A], offset:Long, size:Long) extends AbstractLArrayView[A](base, offset, size) {
113 | protected[this] def newBuilder: LBuilder[A, LArray[A]] = new LObjectArrayBuilder
114 | private[larray] def elementByteSize: Int = 4
115 | }
116 |
117 | object EmptyView extends LArrayView[Nothing] {
118 | protected[this] def newBuilder = EmptyArray.newBuilder
119 | def size = 0L
120 | def apply(i: Long) = EmptyArray.apply(i)
121 |
122 | def address = LArray.EmptyArray.address
123 |
124 | /**
125 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
126 | */
127 | private[larray] def elementByteSize = EmptyArray.elementByteSize
128 |
129 | /**
130 | * Copy the contents of this LSeq[A] into the target LByteArray
131 | * @param dst
132 | * @param dstOffset
133 | */
134 | def copyTo(dst: LByteArray, dstOffset: Long) { EmptyArray.copyTo(dst, dstOffset)}
135 |
136 | /**
137 | * Copy the contents of this sequence into the target LByteArray
138 | * @param srcOffset
139 | * @param dst
140 | * @param dstOffset
141 | * @param blen the byte length to copy
142 | */
143 | def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { EmptyArray.copyTo(srcOffset, dst, dstOffset, blen)}
144 | }
145 |
146 | }
147 |
148 | /**
149 | * Shallow-copy reference of the part of LArray
150 | * @tparam A
151 | */
152 | trait LArrayView[A] extends LSeq[A] {
153 |
154 | }
155 |
156 | abstract class AbstractLArrayView[A : ClassTag](base:LSeq[A], offset:Long, val size:Long) extends LArrayView[A] {
157 |
158 | def address = base.address + (offset * elementByteSize)
159 |
160 | /**
161 | * Retrieve an element
162 | * @param i index
163 | * @return the element value
164 | */
165 | def apply(i: Long): A = base.apply(i + offset)
166 |
167 | /**
168 | * Copy the contents of this LSeq[A] into the target LByteArray
169 | * @param dst
170 | * @param dstOffset
171 | */
172 | def copyTo(dst: LByteArray, dstOffset: Long) {
173 | base.copyTo(offset, dst, dstOffset, byteLength)
174 | }
175 |
176 | /**
177 | * Copy the contents of this sequence into the target LByteArray
178 | * @param srcOffset
179 | * @param dst
180 | * @param dstOffset
181 | * @param blen the byte length to copy
182 | */
183 | def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) {
184 | base.copyTo[B](offset + srcOffset, dst, dstOffset, blen)
185 | }
186 |
187 |
188 |
189 | }
190 |
191 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LBitArray.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LBitArray.scala
19 | // Since: 2012/06/19 3:42 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.{File, FileOutputStream}
26 | import java.nio.ByteBuffer
27 |
28 | import wvlet.log.LogSupport
29 | import xerial.larray.buffer.Memory
30 |
31 | /**
32 | * Helper methods for packing bit sequences into Array[Long]
33 | *
34 | * @author leo
35 | */
36 | object BitEncoder {
37 |
38 | def minArraySize(numBits: Long): Long = {
39 | val blockBitSize: Long = 64L
40 | val arraySize: Long = (numBits + blockBitSize - 1L) / blockBitSize
41 | arraySize
42 | }
43 |
44 | @inline def blockAddr(pos: Long): Long = blockIndex(pos) << 3
45 | @inline def blockIndex(pos: Long): Long = (pos >>> 6)
46 | @inline def blockOffset(pos: Long): Int = (pos & 0x3FL).toInt
47 |
48 | val table = Array(false, true)
49 | }
50 |
51 | /**
52 | * Utilities to build LBitArray
53 | */
54 | object LBitArray {
55 |
56 | def newBuilder = new LBitArrayBuilder()
57 |
58 | def newBuilder(sizeHint: Long) = new LBitArrayBuilder()
59 |
60 | def apply(bitString: String): LBitArray = {
61 | val b = newBuilder
62 | b.sizeHint(bitString.length)
63 | for (ch <- bitString) {
64 | b += (if (ch == '0') {
65 | true
66 | }
67 | else {
68 | false
69 | })
70 | }
71 | b.result()
72 | }
73 |
74 | }
75 |
76 | trait LBitArrayOps {
77 |
78 | /**
79 | * Count the number of bits within the specified range [start, end)
80 | *
81 | * @param checkTrue count true or false
82 | * @param start
83 | * @param end
84 | * @return the number of occurrences
85 | */
86 | def count(checkTrue: Boolean, start: Long, end: Long): Long
87 |
88 | /**
89 | * Extract a slice of the sequence [start, end)
90 | *
91 | * @param start
92 | * @param end
93 | * @return
94 | */
95 | def slice(start: Long, end: Long): LBitArray
96 |
97 | }
98 |
99 | /**
100 | *
101 | * Specialized implementaiton of LArray[Boolean] using LArray[Long]
102 | * To generate an instance of LBitArray, use ``LBitArray.newBuilder(Long)`` or [[xerial.larray.LBitArray#apply]]
103 | *
104 | * @param seq raw bit string
105 | * @param numBits
106 | */
107 | class LBitArray(private[larray] val seq: LLongArray, private val numBits: Long) extends LArray[Boolean] with UnsafeArray[Boolean] with LBitArrayOps {
108 |
109 | self =>
110 |
111 | import BitEncoder._
112 | import UnsafeUtil.unsafe
113 |
114 | private[larray] def alloc = seq.alloc
115 |
116 | def this(numBits: Long) = this(new LLongArray(BitEncoder.minArraySize(numBits)), numBits)
117 | def this(numBits: Long, m: Memory) = this(new LLongArray(BitEncoder.minArraySize(numBits), m), numBits)
118 |
119 | def size = numBits
120 | private var hash: Int = 0
121 |
122 | override def byteLength = seq.byteLength
123 |
124 | protected[this] def newBuilder: LBuilder[Boolean, LBitArray] = new LBitArrayBuilder()
125 | private[larray] def m: Memory = seq.m
126 |
127 | override def toString = {
128 | val displaySize = math.min(500L, numBits)
129 | val b = new StringBuilder
130 | for (i <- 0L until displaySize) {
131 | b.append(if (self(i)) {
132 | "1"
133 | }
134 | else {
135 | "0"
136 | })
137 | }
138 | b.result()
139 | }
140 |
141 | def on(index: Long) = update(index, true)
142 | def off(index: Long) = update(index, false)
143 |
144 | /**
145 | * Set all bits to 1
146 | *
147 | * @return
148 | */
149 | def fill {
150 | unsafe.setMemory(seq.m.address, byteLength, 0xFF.toByte)
151 | }
152 |
153 | /**
154 | * Return the DNA base at the specified index
155 | *
156 | * @param index
157 | * @return
158 | */
159 | def apply(index: Long): Boolean = {
160 | val addr = blockAddr(index)
161 | val offset = blockOffset(index)
162 | val code = ((seq.getLong(addr) >>> offset) & 1L).toInt
163 | table(code)
164 | }
165 |
166 | def update(index: Long, v: Boolean): Boolean = {
167 | val addr = blockAddr(index)
168 | val offset = blockOffset(index)
169 | if (v) {
170 | seq.putLong(addr, seq.getLong(addr) | (1L << offset))
171 | }
172 | else {
173 | seq.putLong(addr, seq.getLong(addr) & (~(1L << offset)))
174 | }
175 | v
176 | }
177 |
178 | private def numFilledBlocks = (size / 64L)
179 |
180 | private def lastBlock = seq.last & (~0L << blockOffset(size))
181 |
182 | override def hashCode() = {
183 | if (hash == 0) {
184 | var h = numBits * 31L
185 | var pos = 0L
186 | for (i <- (0L until numFilledBlocks)) {
187 | h += seq(pos) * 31L
188 | pos += 1
189 | }
190 | if (blockOffset(size) > 0) {
191 | h += lastBlock * 31L
192 | }
193 | hash = h.toInt
194 | }
195 | hash
196 | }
197 |
198 | override def equals(obj: Any): Boolean = {
199 | obj match {
200 | case other: LBitArray => {
201 | if (this.size != other.size) {
202 | false
203 | }
204 | else {
205 | (0L until numFilledBlocks).find(i => this.seq(i) != other.seq(i)) match {
206 | case Some(x) => false
207 | case None => this.lastBlock == other.lastBlock
208 | }
209 | }
210 | }
211 | case _ => false
212 | }
213 | }
214 |
215 | protected def fastCount(v: Long, checkTrue: Boolean): Long = {
216 | // JVM optimizer is smart enough to replace this code to a pop count operation available in the CPU
217 | val c = java.lang.Long.bitCount(v)
218 | if (checkTrue) {
219 | c
220 | }
221 | else {
222 | 64L - c
223 | }
224 | }
225 |
226 | /**
227 | * Count the number of bits within the specified range [start, end)
228 | *
229 | * @param checkTrue count true or false
230 | * @param start
231 | * @param end
232 | * @return the number of occurrences
233 | */
234 | def count(checkTrue: Boolean, start: Long, end: Long): Long = {
235 |
236 | val sPos = blockIndex(start)
237 | val sOffset = blockOffset(start)
238 |
239 | val ePos = blockIndex(end - 1L)
240 | val eOffset = blockOffset(end - 1L)
241 |
242 | var count = 0L
243 | var num0sInMaskedRegion = 0L
244 | var pos = sPos
245 | while (pos <= ePos) {
246 | var mask: Long = ~0L
247 | if (pos == sPos) {
248 | mask <<= (sOffset << 1L)
249 | num0sInMaskedRegion += sOffset
250 | }
251 | if (pos == ePos) {
252 | val rMask = ~0L >>> (62L - (eOffset << 1))
253 | mask &= rMask
254 | num0sInMaskedRegion += 31L - eOffset
255 | }
256 | // Applying bit mask changes all bits to 0s, so we need to subtract num0s
257 | val v: Long = seq(pos) & mask
258 | val popCount = fastCount(v, checkTrue)
259 | count += popCount
260 | pos += 1
261 | }
262 |
263 | if (checkTrue) {
264 | count - num0sInMaskedRegion
265 | }
266 | else {
267 | count
268 | }
269 | }
270 |
271 | /**
272 | * Extract a slice of the sequence [start, end)
273 | *
274 | * @param start
275 | * @param end
276 | * @return
277 | */
278 | override def slice(start: Long, end: Long): LBitArray = {
279 | if (start > end) {
280 | sys.error("illegal argument start:%,d > end:%,d".format(start, end))
281 | }
282 |
283 | val sliceLen = end - start
284 | val newSeq = new LLongArray(minArraySize(sliceLen))
285 | newSeq.clear
286 |
287 | var i = 0L
288 | while (i < sliceLen) {
289 | val sPos = blockIndex(start + i)
290 | val sOffset = blockOffset(start + i)
291 |
292 | val dPos = blockIndex(i)
293 | val dOffset = blockOffset(i)
294 |
295 | var copyLen = 0L
296 | var l = 0L
297 | val v = seq(sPos) & (~0L << sOffset)
298 | if (sOffset == dOffset) {
299 | // no shift
300 | copyLen = 64L
301 | l = v
302 | }
303 | else if (sOffset < dOffset) {
304 | // left shift
305 | val shiftLen = dOffset - sOffset
306 | copyLen = 64L - dOffset
307 | l = v << shiftLen
308 | }
309 | else {
310 | // right shift
311 | val shiftLen = sOffset - dOffset
312 | copyLen = 64L - sOffset
313 | l = v >>> shiftLen
314 | }
315 | newSeq(dPos) |= l
316 | i += copyLen
317 | }
318 |
319 | new LBitArray(newSeq, sliceLen)
320 | }
321 |
322 | /**
323 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
324 | */
325 | private[larray] def elementByteSize: Int =
326 | throw new UnsupportedOperationException("elementByteSize of LBitArray")
327 |
328 | def view(from: Long, to: Long) = new LArrayView.LBitArrayView(this, from, to - from)
329 |
330 | /**
331 | * Save to a file.
332 | *
333 | * @param f
334 | * @return
335 | */
336 | override def saveTo(f: File): File = {
337 | val fout = new FileOutputStream(f).getChannel
338 | try {
339 | // LBitArray need to record numBits
340 | val b = new Array[Byte](8)
341 | val bb = ByteBuffer.wrap(b).putLong(numBits)
342 | bb.flip()
343 | fout.write(bb)
344 | fout.write(this.toDirectByteBuffer)
345 | f
346 | }
347 | finally
348 | fout.close
349 | }
350 |
351 | }
352 |
353 | /**
354 | * BitVector builder
355 | */
356 | class LBitArrayBuilder extends LBuilder[Boolean, LBitArray] with LogSupport {
357 |
358 | import BitEncoder._
359 |
360 | def elementSize = throw new UnsupportedOperationException("elementSize of LBitArrayBuilder")
361 |
362 | private var elems : LLongArray = _
363 | private var capacity: Long = 0L
364 | private var numBits : Long = 0L
365 |
366 | protected def mkArray(size: Long): LLongArray = {
367 | val newArray = new LLongArray(minArraySize(size))
368 | newArray.clear()
369 | if (capacity > 0L) {
370 | LArray.copy(elems, newArray)
371 | elems.free
372 | }
373 | newArray
374 | }
375 |
376 | def sizeHint(size: Long) {
377 | if (capacity < size) resize(size)
378 | }
379 |
380 | protected def ensureSize(size: Long) {
381 | val factor = 2L
382 | if (capacity < size || capacity == 0L) {
383 | var newsize = if (capacity <= 1L) {
384 | 64L
385 | }
386 | else {
387 | capacity * factor
388 | }
389 | while (newsize < size) newsize *= factor
390 | resize(newsize)
391 | }
392 | }
393 |
394 | protected def resize(size: Long) {
395 | elems = mkArray(size)
396 | capacity = size
397 | }
398 |
399 | def clear() {
400 | if (numBits > 0) {
401 | elems.free
402 | }
403 | capacity = 0L
404 | numBits = 0L
405 | }
406 |
407 | /**
408 | * Append a bit value
409 | *
410 | * @param v
411 | */
412 | override def +=(v: Boolean): this.type = {
413 | ensureSize(numBits + 1)
414 | val pos = blockIndex(numBits)
415 | val offset = blockOffset(numBits)
416 | if (v) {
417 | elems(pos) |= (1L << offset)
418 | }
419 | else {
420 | elems(pos) &= ~(1L << offset)
421 | }
422 | numBits += 1
423 | this
424 | }
425 |
426 | def result(): LBitArray = {
427 | val s = minArraySize(numBits)
428 | if (capacity != 0L && capacity == s) {
429 | new LBitArray(elems, numBits)
430 | }
431 | else {
432 | new LBitArray(mkArray(numBits), numBits)
433 | }
434 | }
435 |
436 | def result(numBits: Long): LBitArray = {
437 | this.numBits = numBits
438 | result()
439 | }
440 |
441 | override def toString = result.toString
442 |
443 | def append(seq: LSeq[Boolean]) = {
444 | val N = seq.length
445 | ensureSize(numBits + N)
446 | var i = 0L
447 | while (i < N) {
448 | +=(seq(i))
449 | i += 1
450 | }
451 | numBits += N
452 | this
453 | }
454 |
455 | def write(src: ByteBuffer) = throw new UnsupportedOperationException("write(ByteBuffer)")
456 |
457 | def isOpen = true
458 |
459 | def close() {}
460 | }
461 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/LByteBuffer.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LByteBuffer.scala
19 | // Since: 2013/03/26 14:36
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.nio.{InvalidMarkException, Buffer}
26 |
27 | /**
28 | * ByteBuffer interface of [[xerial.larray.LArray]]
29 | * @author Taro L. Saito
30 | */
31 | class LByteBuffer(buf:LByteArray, private var cursor:Long, private var bufLimit:Long) {
32 | private var _mark : Long = -1L
33 |
34 | def this(buf:LByteArray) = this(buf, 0L, buf.byteLength)
35 |
36 | /**
37 | * Return the LArray representation of this buffer
38 | */
39 | def array : LByteArray = buf
40 |
41 | def position : Long = cursor
42 | def position(newPos:Long) : this.type = { cursor = newPos; this }
43 | def limit : Long = buf.byteLength
44 | def limit(newLimit:Long) : this.type = { bufLimit = newLimit; this }
45 |
46 | def flip : this.type = {
47 | bufLimit = cursor
48 | cursor = 0L
49 | _mark = -1L
50 | this
51 | }
52 |
53 | def rewind : this.type = {
54 | cursor = 0L
55 | _mark = -1L
56 | this
57 | }
58 |
59 | def remaining : Long = bufLimit - cursor
60 | def hasRemaining : Boolean = cursor < bufLimit
61 |
62 | def clear() : this.type = {
63 | cursor = 0L
64 | bufLimit = buf.length
65 | _mark = -1L
66 | this
67 | }
68 |
69 | def mark : this.type = {
70 | _mark = cursor
71 | this
72 | }
73 |
74 | def reset : this.type = {
75 | val m = _mark
76 | if (m < 0L) throw new InvalidMarkException
77 | cursor = m
78 | this
79 | }
80 |
81 | def put(b:Array[Byte]) : this.type = put(b, 0, b.length)
82 | def put(b:Array[Byte], offset:Int, len:Int) : this.type = {
83 | buf.readFromArray(b, offset, cursor, len)
84 | cursor += len
85 | this
86 | }
87 | def put[A](b:LArray[A], offset:Long, len:Long) : this.type = {
88 | b.copyTo(offset, buf, cursor, len)
89 | cursor += len
90 | this
91 | }
92 |
93 | def putBoolean(v:Boolean) : this.type = { buf.putByte(cursor, if(v) 1 else 0); cursor += 1; this }
94 | def putByte(v:Byte) : this.type = { buf.putByte(cursor, v); cursor += 1; this }
95 | def putChar(v:Char) : this.type = { buf.putChar(cursor, v); cursor += 2; this }
96 | def putShort(v:Short) : this.type = { buf.putShort(cursor, v); cursor += 2; this }
97 | def putInt(v:Int) : this.type = { buf.putInt(cursor, v); cursor += 4; this }
98 | def putFloat(v:Float) : this.type = { buf.putFloat(cursor, v); cursor += 4; this }
99 | def putLong(v:Long) : this.type = { buf.putLong(cursor, v); cursor += 8; this }
100 | def putDouble(v:Double) : this.type = { buf.putDouble(cursor, v); cursor += 8; this }
101 |
102 | def get(b:Array[Byte]) : this.type = get(b, 0, b.length)
103 | def get(b:Array[Byte], offset:Int, len:Int) : this.type = {
104 | buf.writeToArray(cursor, b, offset, len)
105 | this
106 | }
107 | def get[A](b:RawByteArray[A], offset:Long, len:Long) : this.type = {
108 | buf.copyTo(cursor, b, offset, len)
109 | cursor += len
110 | this
111 | }
112 | def getBoolean: Boolean = { val v = if(buf.getByte(cursor) == 0) false else true; cursor += 1; v}
113 | def getByte : Byte = { val v = buf.getByte(cursor); cursor += 1; v }
114 | def getChar : Char = { val v = buf.getChar(cursor); cursor += 2; v }
115 | def getShort : Short = { val v = buf.getShort(cursor); cursor += 2; v }
116 | def getInt : Int = { val v = buf.getInt(cursor); cursor += 4; v }
117 | def getFloat : Float = { val v = buf.getFloat(cursor); cursor += 4; v }
118 | def getLong : Long = { val v = buf.getLong(cursor); cursor += 8; v }
119 | def getDouble : Double = { val v = buf.getDouble(cursor); cursor += 8; v }
120 |
121 |
122 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/MappedLByteArray.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // MappedLByteArray.scala
19 | // Since: 2013/04/04 10:47 AM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.File
26 |
27 | import xerial.larray.buffer.{Memory, MemoryAllocator}
28 | import xerial.larray.mmap.{MMapBuffer, MMapMode}
29 |
30 | /**
31 | * Memory-mapped LByteArray
32 | *
33 | * @author Taro L. Saito
34 | */
35 | class MappedLByteArray(f: File, offset: Long = 0, val size: Long = -1, mode: MMapMode = MMapMode.READ_WRITE)(implicit alloc: MemoryAllocator) extends RawByteArray[Byte] {
36 |
37 | import java.{lang => jl}
38 |
39 | import UnsafeUtil.unsafe
40 |
41 | private val mmap = new MMapBuffer(f, offset, size, mode);
42 | private val m: Memory = mmap.m
43 |
44 | protected[this] def newBuilder = new LByteArrayBuilder
45 |
46 | val address = mmap.address()
47 |
48 | def free {
49 | m.release();
50 | }
51 |
52 | /**
53 | * Forces any changes made to this buffer to be written to the file
54 | */
55 | def flush {
56 | mmap.flush()
57 | }
58 |
59 | /**
60 | * Close the memory mapped file. To ensure the written data is saved in the file, call flush before closing
61 | */
62 | override def close() {
63 | mmap.close()
64 | }
65 |
66 | /**
67 | * Update an element
68 | *
69 | * @param i index to be updated
70 | * @param v value to set
71 | * @return the value
72 | */
73 | def update(i: Long, v: Byte) = {unsafe.putByte(address + i, v); v}
74 |
75 | def view(from: Long, to: Long) = new LArrayView.LByteArrayView(this, from, to - from)
76 |
77 | /**
78 | * Retrieve an element
79 | *
80 | * @param i index
81 | * @return the element value
82 | */
83 | def apply(i: Long) = unsafe.getByte(address + i)
84 |
85 | /**
86 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
87 | */
88 | private[larray] def elementByteSize = 1
89 |
90 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/UInt32Array.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // UInt32Array.scala
19 | // Since: 2013/03/18 3:57 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import xerial.larray.buffer.{MemoryAllocator, Memory}
26 |
27 | object UInt32Array {
28 |
29 | def newBuilder = new LArrayBuilder[Long, UInt32Array] {
30 |
31 | def +=(v: Long): this.type = {
32 | ensureSize(numElems + 1)
33 | elems.putInt(cursor, (v & 0xFFFFFFFFL).toInt)
34 | cursor += elementSize
35 | this
36 | }
37 |
38 | /** Produces a collection from the added elements.
39 | * The builder's contents are undefined after this operation.
40 | * @return a collection containing the elements added to this builder.
41 | */
42 | def result(): UInt32Array = {
43 | if(capacity != 0L && capacity == numElems) new UInt32Array(numElems, elems.m)
44 | else new UInt32Array(numElems, mkArray(numElems).m)
45 | }
46 |
47 | def elementSize = 4
48 | }
49 |
50 |
51 |
52 | }
53 |
54 |
55 | private[larray] class UInt32ArrayView(base:UInt32Array, offset:Long, val size:Long) extends LArrayView[Long] {
56 | protected[this] def newBuilder: LBuilder[Long, UInt32Array] = UInt32Array.newBuilder
57 | def apply(i: Long) = base.apply(offset + i)
58 | private[larray] def elementByteSize = 4
59 | def copyTo(dst: LByteArray, dstOffset: Long) { base.copyTo(offset, dst, dstOffset, byteLength) }
60 | def copyTo[B](srcOffset: Long, dst: RawByteArray[B], dstOffset: Long, blen: Long) { base.copyTo(offset+srcOffset, dst, dstOffset, blen) }
61 |
62 | def address = base.address + (offset * elementByteSize)
63 | }
64 |
65 | /**
66 | * Array of uint32 values. The internal array representation is the same with LIntArray, but the apply and update methods are based on Long type values.
67 | *
68 | * @author Taro L. Saito
69 | */
70 | class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryAllocator) extends LArray[Long] with UnsafeArray[Long] { self =>
71 | def this(size:Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc)
72 |
73 | import UnsafeUtil.unsafe
74 |
75 | protected[this] def newBuilder: LBuilder[Long, UInt32Array] = UInt32Array.newBuilder
76 |
77 | def apply(i:Long) : Long = {
78 | val v : Long = unsafe.getInt(m.address + (i << 2)) & 0xFFFFFFFFL
79 | v
80 | }
81 |
82 | def update(i:Long, v:Long) : Long = {
83 | unsafe.putInt(m.address + (i << 2), (v & 0xFFFFFFFFL).toInt)
84 | v
85 | }
86 |
87 | /**
88 | * Byte size of an element. For example, if A is Int, its elementByteSize is 4
89 | */
90 | private[larray] def elementByteSize: Int = 4
91 |
92 |
93 | def view(from: Long, to: Long) : LArrayView[Long] = new UInt32ArrayView(self, from, to - from)
94 |
95 | }
96 |
97 |
98 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/UnsafeUtil.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // UnsafeUtil.scala
19 | // Since: 2013/04/04 5:36 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.nio.ByteBuffer
26 |
27 | import sun.misc.Unsafe
28 | import wvlet.log.LogSupport
29 |
30 | /**
31 | * Utilities for accessing sun.misc.Unsafe
32 | *
33 | * @author Taro L. Saito
34 | */
35 | object UnsafeUtil extends LogSupport {
36 | val unsafe = {
37 | val f = classOf[Unsafe].getDeclaredField("theUnsafe")
38 | f.setAccessible(true)
39 | f.get(null).asInstanceOf[Unsafe]
40 | }
41 |
42 | private val dbbCC = Class.forName("java.nio.DirectByteBuffer").getDeclaredConstructor(classOf[Long], classOf[Int])
43 |
44 | def newDirectByteBuffer(addr: Long, size: Int): ByteBuffer = {
45 | dbbCC.setAccessible(true)
46 | val b = dbbCC.newInstance(new java.lang.Long(addr), new java.lang.Integer(size))
47 | b.asInstanceOf[ByteBuffer]
48 | }
49 |
50 | val byteArrayOffset = unsafe.arrayBaseOffset(classOf[Array[Byte]]).toLong
51 | val objectArrayOffset = unsafe.arrayBaseOffset(classOf[Array[AnyRef]]).toLong
52 | val objectArrayScale = unsafe.arrayIndexScale(classOf[Array[AnyRef]]).toLong
53 | val addressBandWidth = System.getProperty("sun.arch.data.model", "64").toInt
54 | private val addressFactor = if (addressBandWidth == 64) {
55 | 8L
56 | }
57 | else {
58 | 1L
59 | }
60 | val addressSize = unsafe.addressSize()
61 |
62 | /**
63 | * @param obj
64 | * @return
65 | *
66 | */
67 | @deprecated(message = "Deprecated because this method does not return correct object addresses in some platform", since = "0.1")
68 | def getObjectAddr(obj: AnyRef): Long = {
69 | trace(f"address factor:$addressFactor%d, addressSize:$addressSize, objectArrayOffset:$objectArrayOffset, objectArrayScale:$objectArrayScale")
70 |
71 | val o = new Array[AnyRef](1)
72 | o(0) = obj
73 | objectArrayScale match {
74 | case 4 => (unsafe.getInt(o, objectArrayOffset) & 0xFFFFFFFFL) * addressFactor
75 | case 8 => (unsafe.getLong(o, objectArrayOffset) & 0xFFFFFFFFFFFFFFFFL) * addressFactor
76 | }
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/example/LArrayExample.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayExample.scala
19 | // Since: 2013/03/23 23:16
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.example
24 |
25 | /**
26 | * Example of LArray Usages
27 | * @author Taro L. Saito
28 | */
29 | class LArrayExample {
30 |
31 | import xerial.larray._
32 |
33 | // Create a new LArray of Int type of length = 5
34 | val l = LArray.of[Int](5)
35 |
36 | // Create an LArray with initial values
37 | val ll = LArray(3, 5, 9, 10)
38 |
39 | // Set elements. To specify a range of LArray, use 'Until' (for Long range) instead of 'until' (for Int range).
40 | for(i <- 0 Until l.size)
41 | l(i) = i.toInt
42 |
43 | // Read elements
44 | val e0 = l(0)
45 | val e1 = l(1)
46 |
47 | // Print the elements
48 | println(l.mkString(", ")) // 0, 1, 2, 3, 4
49 |
50 | // Traverse the elements with their indexes
51 | for((e, i) <- l.zipWithIndex)
52 | println(s"l($i) = $e") // l(0) = 0, l(1) = 1, ...
53 |
54 | // Manipulate LArray
55 | val l2 = l.map(_ * 10) // LArray(0, 10, 20, 30, 40)
56 | val f = l.filter(_ % 2 == 0) // LArray(0, 2, 4)
57 | val s = l.slice(2) // LArray(2, 3, 4)
58 | l.foreach(println(_))
59 |
60 | // Build LArray
61 | val b = LArray.newBuilder[Int]
62 | for(i <- 0 until (10, step=3))
63 | b += i
64 | val lb = b.result // LArray(0, 3, 6, 9)
65 |
66 | // Convert to Scala Array
67 | val arr = l.toArray
68 | println(arr.mkString(", ")) // 0, 1, 2, 3, 4
69 |
70 | // Convert Scala Array to LArray
71 | val arr2 = Array(1, 3, 5)
72 | val la = arr2.toLArray
73 |
74 | // Save to a file
75 | import java.io.File
76 | val file = l.saveTo(new File("target/larray.tmp"))
77 | file.deleteOnExit()
78 | // Load from a file
79 | val l3 = LArray.loadFrom[Int](file) // LArray(0, 1, 2, 3, 4)
80 |
81 | // Initialize the array
82 | l.clear()
83 | println(l.mkString(", ")) // 0, 0, 0, 0, 0
84 |
85 |
86 | // Release the memory contents.
87 | l.free
88 | l3.free
89 |
90 | // You can omit calling free, because GC collects unused LArrays
91 |
92 | println("done.")
93 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.example;
17 |
18 | import scala.runtime.AbstractFunction1;
19 | import xerial.larray.LArray;
20 | import xerial.larray.LIntArray;
21 | import xerial.larray.LIntArrayBuilder;
22 | import xerial.larray.LIterator;
23 | import xerial.larray.buffer.LBuffer;
24 | import xerial.larray.japi.LArrayJ;
25 |
26 | import java.io.File;
27 |
28 | /**
29 | * LArray example in Java
30 | * @author Taro L. Saito
31 | */
32 | public class LArrayJavaExample {
33 |
34 | public static void main(String[] args) {
35 | // Create a new LArray of Int type
36 | LIntArray l = LArrayJ.newLIntArray(5);
37 |
38 | // Read elements
39 | int e0 = l.apply(0);
40 | int e1 = l.apply(1);
41 |
42 | // Set elements
43 | for(int i=0; i l2 = l.map(new AbstractFunction1(){
58 | public Object apply(Object v1) {
59 | return Integer.class.cast(v1) * 10;
60 | }
61 | });
62 | LIterator f = l.filter(new AbstractFunction1(){
63 | public Object apply(Object v1) {
64 | return Integer.class.cast(v1) % 2 == 0;
65 | }
66 | });
67 | LArray s = l.slice(2);
68 |
69 | // Build LArray
70 | LIntArrayBuilder b = new LIntArrayBuilder();
71 | for(int i=0; i<10; i += 3)
72 | b.append(i);
73 | LIntArray lb = b.result(); // LArray(0, 3, 6, 9)
74 | System.out.println(lb.mkString(", "));
75 |
76 | // Save to a file
77 | File file = l.saveTo(new File("target/larray.tmp"));
78 | file.deleteOnExit();
79 |
80 | // Load from a file
81 | LArray l3 = LArrayJ.loadLIntArrayFrom(file); // LArray(0, 1, 2, 3, 4)
82 |
83 | // Initialize the array
84 | l.clear();
85 | System.out.println(l.mkString(", ")); // 0, 0, 0, 0, 0
86 |
87 | // Release the memory contents
88 | l.free();
89 |
90 |
91 | // Using LBuffer
92 | LBuffer lbuf = new LBuffer(1000);
93 | lbuf.putInt(0, 10);
94 | int ten = lbuf.getInt(0);
95 | lbuf.address(); // memory address
96 | lbuf.release(); // deallocate the memory
97 |
98 |
99 | System.out.println("done.");
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/package.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial
17 |
18 | import reflect.ClassTag
19 | import xerial.larray.buffer.LBufferConfig
20 |
21 | /**
22 | * == LArray ==
23 | * [[xerial.larray.LArray]] is a large off-heap array that can hold more than 2G (2^31) entries.
24 | *
25 | * === Features ===
26 | *
27 | * - Supporting huge array size upto 2^63 -1 entries.
28 | * - 2^31 -1 (2G) is the limitation of the default Java/Scala array size, because 32-bit signed integer (int) is used for the array indexes. To resolve this, LArray uses long type indexes of 64-bit signed integers.
29 | * - For example the entire human genome data (3GB) can be stored in LArray.
30 | * - LArray can be released from the main memory immediately.
31 | * - Call LArray.free to release acquired memory resources.
32 | * - The default arrays in Java/Scala are resident in JVM heaps, in which users cannot free the allocate arrays space even if they become unnecessary. That means it is hard to avoid OutOfMemoryException when working with large amount of data.
33 | * - LArray is in sync with Garbage Collection (GC)
34 | * - Even if you forget to call LArray.free, the acquired memory will be released when GC sweeps LArray instances.
35 | * - To prevent accidental memory release, keep a reference to LArray somewhere (e.g., in List)
36 | * - LArray uses off-heap memory, so it is free from the limitation of JVM memory manager.
37 | * - LArray uses memory space outside of the default JVM heap, so creating LArrays with more than -Xmx(maximum memory size) is possible. This is useful when you need large amount of memory or the amount of memory required in your application is unknown.
38 | * - Fast copy and memory allocation
39 | * - Rich set of operations for LArray[A]
40 | * - map, filter, reduce, zip, etc.
41 | * - See [[xerial.larray.LArray]]
42 | *
43 | * === Limitations ===
44 | *
45 | * - LArray[A] of generic objects (e.g., LArray[String], LArray[AnyRef]) cannot be released immedeately from the main memory, because objects other than primitive types need to be created on JVM heaps and become subject to GC.
46 | * - To release objects from main memory, you need to create *off-heap* objects. For example, create a large `LArray[Byte]`, then align your object data on the array. Object parameters can be retrieved with `LArray[Byte].getInt(offset)`, `getFloat(offset)`, etc.
47 | *
48 | */
49 | package object larray {
50 |
51 | implicit def defaultAllocator : xerial.larray.buffer.MemoryAllocator = LBufferConfig.allocator
52 |
53 |
54 | implicit class ConvertArrayToLArray[A : ClassTag](arr:Array[A]) {
55 | def toLArray : LArray[A] = {
56 | val l = LArray.of[A](arr.length).asInstanceOf[RawByteArray[A]]
57 | var i = 0
58 | while(i < arr.length) {
59 | l(i) = arr(i)
60 | i += 1
61 | }
62 | l
63 | }
64 | }
65 |
66 | implicit class ConvertIterableToLArray[A : ClassTag](it:Iterable[A]) {
67 | def toLArray : LArray[A] = {
68 | val b = LArray.newBuilder[A]
69 | it.foreach(b += _)
70 | b.result
71 | }
72 | }
73 |
74 | implicit class AsLRange(from:Long) {
75 | def Until(to:Long) : LIterator[Long] = Until(to, 1)
76 | def Until(to:Long, step:Long) : LIterator[Long] = new AbstractLIterator[Long] {
77 | private var cursor = from
78 | def hasNext = cursor < to
79 | def next() = {
80 | val v = cursor
81 | cursor += step
82 | v
83 | }
84 | }
85 |
86 |
87 | def To(to:Long) : LIterator[Long] = To(to, 1)
88 |
89 | def To(to:Long, step:Long) : LIterator[Long] = new AbstractLIterator[Long] {
90 | private var cursor = from
91 | def hasNext = cursor <= to
92 | def next() = {
93 | val v = cursor
94 | cursor += step
95 | v
96 | }
97 | }
98 |
99 | }
100 |
101 |
102 |
103 | }
--------------------------------------------------------------------------------
/larray/src/main/scala/xerial/larray/util/Logger.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // Logger.scala
19 | // Since: 2013/03/22 2:32
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.util
24 |
25 | /**
26 | * Logger wrapper for using `wvlet.log.Logger` in Java
27 | *
28 | * @author Taro L. Saito
29 | */
30 | class Logger(cl:Class[_]) {
31 | private val _logger = wvlet.log.Logger.apply(cl.getName)
32 |
33 | def trace(m:String) { _logger.trace(m) }
34 | def debug(m:String) { _logger.debug(m) }
35 | def info(m:String) { _logger.info(m) }
36 | def warn(m:String) { _logger.warn(m) }
37 | def error(m:String) { _logger.error(m) }
38 | def fatal(m:String) { _logger.error(m) }
39 |
40 | }
--------------------------------------------------------------------------------
/larray/src/multi-jvm/scala/xerial/larray/SharedMemory.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray
17 |
18 | import java.io.{File, FileFilter}
19 |
20 | import wvlet.log.LogSupport
21 | import xerial.larray.mmap.MMapMode
22 |
23 | object SharedMemoryTest {
24 |
25 | }
26 |
27 | trait Barrier extends LogSupport {
28 | this: LArraySpec =>
29 |
30 | val numJVMs: Int
31 |
32 | val jvmID = {
33 | val n = getClass.getSimpleName
34 | val p = "[0-9]+".r
35 | val id = p.findAllIn(n).toSeq.last.toInt
36 | info(s"jvm ID:$id")
37 | id
38 | }
39 |
40 | def enterBarrier(name: String) {
41 | debug(s"entering barrier:$name")
42 | val f = new File(s"target/${name}.barrier")
43 | f.deleteOnExit()
44 | val m = LArray.mmap(f, 0, numJVMs, MMapMode.READ_WRITE)
45 | m(jvmID - 1) = 1.toByte
46 | m.flush
47 |
48 | def reached(): Boolean = {
49 | m.forall(_ == 1.toByte)
50 | }
51 |
52 | while (!reached()) {
53 | Thread.sleep(10)
54 | }
55 | debug(s"exit barrier: $name")
56 | }
57 |
58 | before {
59 | if (jvmID == 1) {
60 | val lockFile = Option(new File("target").listFiles(new FileFilter {
61 | def accept(pathname: File) = pathname.getName.endsWith(s".barrier")
62 | })) getOrElse (Array.empty[File])
63 |
64 | while (lockFile.exists(_.exists())) {
65 | lockFile.filter(_.exists()) map (_.delete())
66 | Thread.sleep(50)
67 | }
68 | }
69 | else {
70 | Thread.sleep(1000)
71 | }
72 | }
73 |
74 | }
75 |
76 | trait SharedMemorySpec extends LArraySpec with Barrier {
77 | val numJVMs = 2
78 |
79 | }
80 |
81 | class SharedMemoryMultiJvm1 extends SharedMemorySpec {
82 |
83 | "mmap" should {
84 |
85 | "be able to shared between processes" in {
86 |
87 | info("preparing mmap")
88 |
89 | //val d = new File("/dev/shm")
90 | val f = new File(new File("target"), "sharedmemory.mmap")
91 | //f.deleteOnExit
92 | val m = LArray.mmap(f, 0, 100, MMapMode.READ_WRITE)
93 | for (i <- 0 Until m.size) {
94 | m(i) = i.toByte
95 | }
96 |
97 | m.flush
98 | debug(m.mkString(","))
99 |
100 | for (i <- 0 Until m.size) {
101 | if ((i % 64) == 0) {
102 | m(i) = 1.toByte
103 | }
104 | else {
105 | m(i) = 0.toByte
106 | }
107 | }
108 |
109 | enterBarrier("prepare")
110 |
111 | //m.close
112 | info("done.")
113 | }
114 | }
115 | }
116 |
117 | class SharedMemoryMultiJvm2 extends SharedMemorySpec {
118 | "mmap" should {
119 | "be able to shared between processes" in {
120 |
121 | enterBarrier("prepare")
122 |
123 | val f = new File(new File("target"), "sharedmemory.mmap")
124 | val m = LArray.mmap(f, 0, 100, MMapMode.READ_WRITE)
125 |
126 | debug(m.mkString(","))
127 |
128 | m(0) shouldBe (1.toByte)
129 | m(1) shouldBe (0.toByte)
130 | m(64) shouldBe (1.toByte)
131 | //m.close
132 | }
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/larray/src/test/java/xerial/larray/japi/JLArrayTest.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.japi;
17 |
18 | import junit.framework.Assert;
19 | import org.junit.Test;
20 | import xerial.larray.LIntArray;
21 | import xerial.larray.util.Logger;
22 |
23 | /**
24 | * @author Taro L. Saito
25 | */
26 | public class JLArrayTest {
27 |
28 | Logger _logger = new Logger(this.getClass());
29 |
30 | @Test
31 | public void constructor() {
32 |
33 | LIntArray l = LArrayJ.newLIntArray(5L);
34 | for (long i = 0; i < l.size(); ++i) l.update(i, (int) i * 2);
35 | _logger.debug(l.mkString(", "));
36 | for (long i = 0; i < l.size(); ++i) l.update(i, (int) (i * i));
37 | _logger.debug(l.mkString(", "));
38 |
39 | Assert.assertEquals(5L, l.size());
40 |
41 | l.free();
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/larray/src/test/java/xerial/larray/japi/JSnappyCompressTest.java:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray.japi;
17 |
18 | import org.junit.Test;
19 | import org.xerial.snappy.Snappy;
20 | import xerial.larray.LByteArray;
21 | import xerial.larray.LIntArray;
22 | import xerial.larray.MappedLByteArray;
23 | import xerial.larray.buffer.LBuffer;
24 | import xerial.larray.mmap.MMapMode;
25 | import xerial.larray.util.Logger;
26 |
27 | import java.io.File;
28 |
29 | public class JSnappyCompressTest {
30 |
31 | private Logger _logger = new Logger(this.getClass());
32 |
33 |
34 |
35 | @Test
36 | public void testCompress() throws Exception {
37 |
38 | long N = 100000000L;
39 | LIntArray l = LArrayJ.newLIntArray(N);
40 | for (int i = 0; i < N; i++) {
41 | l.update(i, (int) (Math.toDegrees(Math.sin(i / 360))));
42 | }
43 | for (int iter = 0; iter < 10; iter++) {
44 | long sum = 0;
45 | long start = System.currentTimeMillis();
46 | for (int i = 0; i < l.length(); i++) {
47 | sum += l.apply(i);
48 | }
49 | long end = System.currentTimeMillis();
50 | _logger.debug("time:" + (end - start) + " sum:" + sum);
51 | }
52 | _logger.debug("compressing the data");
53 | int maxLen = Snappy.maxCompressedLength((int) l.byteLength());
54 | LByteArray compressedBuf = LArrayJ.newLByteArray(maxLen);
55 | long compressedLen = Snappy.rawCompress(l.address(), l.byteLength(),
56 | compressedBuf.address());
57 | LByteArray compressed = (LByteArray) compressedBuf.slice(0,
58 | compressedLen);
59 | File f = File.createTempFile("snappy", ".dat", new File("target"));
60 | f.deleteOnExit();
61 | compressed.saveTo(f);
62 |
63 | _logger.debug("decompressing the data");
64 | long T1 = System.currentTimeMillis();
65 | MappedLByteArray b = LArrayJ.mmap(f, 0, f.length(), MMapMode.READ_ONLY);
66 | long T2 = System.currentTimeMillis();
67 |
68 | long len = Snappy.uncompressedLength(b.address(), b.length());
69 | LIntArray decompressed = LArrayJ.newLIntArray(len / 4);
70 | Snappy.rawUncompress(b.address(), b.length(), decompressed.address());
71 | long T3 = System.currentTimeMillis();
72 | _logger.debug("Map time:" + (T2 - T1));
73 | _logger.debug("decompress time:" + (T3 - T2));
74 |
75 |
76 | _logger.debug("Summing decompressed array");
77 | for (int iter = 0; iter < 10; iter++) {
78 | long sum = 0;
79 | long start = System.currentTimeMillis();
80 | for (int i = 0; i < decompressed.length(); i++) {
81 | sum += decompressed.apply(i);
82 | }
83 | long end = System.currentTimeMillis();
84 | _logger.debug("time:" + (end - start) + " sum:" + sum);
85 | }
86 | }
87 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/IOPerfTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // IOPerfTest.scala
19 | // Since: 2013/03/26 23:27
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io._
26 |
27 | import wvlet.log.io.IOUtil
28 |
29 | import scala.util.Random
30 |
31 | /**
32 | * @author Taro L. Saito
33 | */
34 | class IOPerfTest extends LArraySpec {
35 |
36 | def createSampleFile: File = {
37 | val file = File.createTempFile("sample", ".larray", new File("target"))
38 | file.deleteOnExit()
39 | val b = new Array[Byte](1024 * 1024)
40 | //val P = 1024 * 1024
41 | val P = 64
42 | val f = new FileOutputStream(file)
43 | for (i <- 0 until P) {
44 | Random.nextBytes(b)
45 | f.write(b)
46 | }
47 | f.close
48 | file
49 | }
50 |
51 | "LArray" should {
52 | "compare I/O performance" in {
53 | val f1 = createSampleFile
54 | time("read", repeat = 10) {
55 | block("LArray.loadFrom") {
56 | trace("Loading to LArray")
57 | val l = LArray.loadFrom[Byte](f1)
58 | l.free
59 | }
60 | block("FileOutputStream") {
61 | trace("Loading to Array")
62 | IOUtil.readFully(new BufferedInputStream(new FileInputStream(f1))) {buf =>
63 | // do nothing
64 | }
65 | }
66 | }
67 | }
68 | }
69 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LArrayBuilderTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayBuilderTest.scala
19 | // Since: 2013/03/19 10:37 AM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import scala.util.Random
26 |
27 | /**
28 | * @author Taro L. Saito
29 | */
30 | class LArrayBuilderTest extends LArraySpec {
31 |
32 | "LArrayBuilder" should {
33 |
34 | "build LArray" in {
35 |
36 | val b = LArray.newBuilder[Int]
37 |
38 | def elem(i:Long) = math.toDegrees(math.sin(i / 15.0)).toInt
39 |
40 | for(i <- 0L until 100L)
41 | b += elem(i)
42 |
43 | val l = b.result
44 | debug(l.mkString(", "))
45 | l.size should be (100)
46 | l.zipWithIndex.forall {case (v, i) => v == elem(i) } should be (true)
47 | }
48 |
49 | "build large LArray" in {
50 |
51 | val b = LArray.newBuilder[Byte]
52 | val N = 3L * 1024 * 1024
53 | debug("Creating large array")
54 | val r = new Random(0)
55 | var i = 0L
56 | while(i < N) {
57 | b += r.nextInt(255).toByte
58 | i += 1
59 | }
60 | val l = b.result
61 | l.size should be (N)
62 |
63 | debug("Checking the elements")
64 | val r2 = new Random(0)
65 | l.forall(v => v == r2.nextInt(255).toByte) should be (true)
66 | }
67 |
68 |
69 | }
70 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LArrayInputStreamTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayInputStreamTest.scala
19 | // Since: 2013/03/21 23:38
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import wvlet.log.io.IOUtil
26 |
27 | class LArrayInputStreamTest extends LArraySpec {
28 |
29 | "LArrayInputStream" should {
30 | "be created from LArray[A]" in {
31 | val l = LArray(1, 3, 4, 5)
32 | debug(s"input ${l.mkString(", ")}")
33 | val in = LArrayInputStream(l)
34 | IOUtil.readFully(in) {buf =>
35 | debug(s"buf length: ${buf.length}")
36 | val out = new LArrayOutputStream[Int]
37 | out.write(buf)
38 | val r = out.result
39 | debug(s"output ${r.mkString(", ")}")
40 | l.sameElements(r) should be(true)
41 | }
42 | }
43 | }
44 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LArraySpec.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // GenomeWeaverSpecverSpec.scala
19 | // Since: 2013/03/01 10:55
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import org.scalatest._
26 | import java.io.{ByteArrayOutputStream, PrintStream}
27 |
28 | import wvlet.log.LogFormatter.SourceCodeLogFormatter
29 | import wvlet.log.{LogSupport, Logger}
30 | import wvlet.log.io.ResourceReader
31 | import wvlet.log.io.Timer
32 |
33 | import scala.language.implicitConversions
34 |
35 |
36 | /**
37 | * @author leo
38 | */
39 | trait LArraySpec extends WordSpec with Matchers with ResourceReader with Timer with LogSupport with BeforeAndAfterAll with BeforeAndAfter with GivenWhenThen with BeforeAndAfterEach {
40 |
41 | implicit def toTag(t:String) = Tag(t)
42 |
43 | /**
44 | * Captures the output stream and returns the printed messages as a String
45 | * @param body
46 | * @tparam U
47 | * @return
48 | */
49 | def captureOut[U](body: => U) : String = {
50 | val out = new ByteArrayOutputStream
51 | Console.withOut(out) {
52 | body
53 | }
54 | new String(out.toByteArray)
55 | }
56 |
57 | /**
58 | * Captures the output stream and returns the printed messages as a String
59 | * @param body
60 | * @tparam U
61 | * @return
62 | */
63 | def captureSystemOut[U](body: => U) : String = {
64 | val prev = System.out
65 | val b = new ByteArrayOutputStream
66 | val out = new PrintStream(b)
67 | try {
68 | System.setOut(out)
69 | body
70 | out.flush()
71 | }
72 | finally
73 | System.setOut(prev)
74 | new String(b.toByteArray)
75 | }
76 |
77 | def captureErr[U](body: => U) : String = {
78 | val out = new ByteArrayOutputStream
79 | Console.withErr(out) {
80 | body
81 | }
82 | new String(out.toByteArray)
83 | }
84 |
85 | Logger.setDefaultFormatter(SourceCodeLogFormatter)
86 |
87 | override protected def beforeAll(): Unit = {
88 | // Run LogLevel scanner (log-test.properties or log.properties in classpath) every 1 minute
89 | Logger.scheduleLogLevelScan
90 | super.beforeAll()
91 | }
92 |
93 | override protected def afterAll(): Unit = {
94 | Logger.stopScheduledLogLevelScan
95 | super.afterAll()
96 | }
97 |
98 | }
99 |
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LArrayTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayTest.scala
19 | // Since: 2013/03/13 13:46
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import scala.util.Random
26 | import java.io.{FileInputStream, File, FileOutputStream}
27 |
28 | /**
29 | * @author Taro L. Saito
30 | */
31 | class LArrayTest extends LArraySpec {
32 |
33 | val G: Long = 1024L * 1024L * 1024L
34 |
35 | override def afterEach {
36 | // MemoryAllocator.default.releaseAll
37 | // System.gc()
38 | }
39 |
40 | "LArray" should {
41 | "have constructor" taggedAs("cc") in {
42 |
43 | val l = LArray(1, 2, 3)
44 | val l0 = LArray()
45 | val l1 = LArray(1)
46 |
47 | try {
48 | l.size should be(3)
49 | l(0) should be(1)
50 | l(1) should be(2)
51 | l(2) should be(3)
52 |
53 | l0.size should be(0)
54 | try {
55 | l0.apply(3)
56 | fail("cannot reach here")
57 | }
58 | catch {
59 | case e: Exception => // "OK"
60 | }
61 | }
62 | finally {
63 | l.free
64 | l0.free
65 | l1.free
66 | }
67 | }
68 |
69 | "have map/flatMap" taggedAs("map") in {
70 | val l = LArray(1, 3, 5)
71 | def mul(v: Int) = v * 2
72 |
73 | val m = l.map(mul).toArray
74 | m.size should be(3)
75 | for (i <- 0L until l.size)
76 | m(i.toInt) should be(mul(l(i)))
77 | }
78 |
79 | "read/write values correctly" in {
80 | info("read/write test")
81 |
82 | val step = 1L
83 | val l = new LIntArray((0.1 * G).toLong)
84 | try {
85 | def v(i: Long) = (i * 2).toInt
86 | for (i <- 0L until(l.size, step)) l(i) = v(i)
87 | def loop(i: Long): Boolean = {
88 | if (i >= l.size)
89 | true
90 | else
91 | l(i) == v(i) && loop(i + step)
92 | }
93 |
94 | loop(0) should be(true)
95 | }
96 | finally {
97 | l.free
98 | }
99 | }
100 |
101 | "read/write data to Array[Byte]" taggedAs ("rw") in {
102 | val l = LArray(1, 3)
103 | val b = new Array[Byte](l.byteLength.toInt)
104 |
105 | l match {
106 | case l: RawByteArray[_] =>
107 | debug(s"LArray: [${l.mkString(", ")}]")
108 | debug(s"Array[Byte]: [${b.mkString(", ")}]")
109 | l.writeToArray(0, b, 0, l.byteLength.toInt)
110 | case _ => fail("cannot reach here")
111 | }
112 |
113 | debug(s"Array[Byte]: [${b.mkString(", ")}]")
114 | val l2 = LArray(0, 0)
115 | l2 match {
116 | case l2: RawByteArray[_] =>
117 | l2.readFromArray(b, 0, 0, b.length)
118 | debug(s"LArray2: [${l2.mkString(", ")}]")
119 | l.sameElements(l2) should be(true)
120 | case _ => fail("cannot reach here")
121 | }
122 |
123 |
124 | }
125 |
126 | "read/write data through ByteBuffer" taggedAs ("bb") in {
127 | val l = LArray(1, 3, 5, 134, 34, -3, 2)
128 | debug(l.mkString(", "))
129 | val bb = l.toDirectByteBuffer
130 |
131 | val tmp = File.createTempFile("larray-dump", ".dat", new File("target"))
132 | debug(s"Write LArray to file: $tmp")
133 | tmp.deleteOnExit()
134 | val out = new FileOutputStream(tmp).getChannel
135 | out.write(bb)
136 | out.close()
137 |
138 | // read LArray from file
139 | {
140 | val in = new FileInputStream(tmp).getChannel
141 | val fileSize = in.size()
142 | val newArray = new LByteArray(fileSize)
143 |
144 | var pos = 0L
145 | while (pos < fileSize) {
146 | pos += in.transferTo(pos, fileSize - pos, newArray)
147 | }
148 | val l2 = LArray.wrap[Int](newArray.byteLength, newArray.m)
149 | debug(l2.mkString(", "))
150 |
151 | l.sameElements(l2) should be(true)
152 |
153 | in.close
154 | }
155 |
156 | // read LArray from File using LArrayBuilder
157 | {
158 | val ib = LArray.newBuilder[Int]
159 | val in = new FileInputStream(tmp).getChannel
160 | val fileSize = in.size()
161 | var pos = 0L
162 | while(pos < fileSize) {
163 | pos += in.transferTo(pos, fileSize - pos, ib)
164 | }
165 | val l2 = ib.result()
166 | debug(l2.mkString(", "))
167 | l.sameElements(l2) should be (true)
168 |
169 | in.close
170 | }
171 |
172 | }
173 |
174 | "provide initializer" taggedAs("init") in {
175 |
176 | {
177 | val l = new LIntArray(1000)
178 | l(40) = 34
179 | l.clear()
180 | l.forall(_ == 0) should be (true)
181 | }
182 |
183 |
184 | {
185 | val l = new LByteArray(1000)
186 | l(340) = 34.toByte
187 | l.clear()
188 | l.forall(_ == 0) should be (true)
189 | }
190 | }
191 |
192 | "compare its random access performance with native Scala array and its wrapper" in {
193 | //val N = 1 * 1024 * 1024 * 1024
194 | val N = 1 * 1024 * 1024
195 | info("benchmark has started..")
196 | val arr1 = new Array[Int](N)
197 | val arr2 = new LIntArray(N)
198 | val arr3 = new LIntArraySimple(N)
199 | val arr4 = new MatrixBasedLIntArray(N)
200 |
201 | try {
202 | val r = new Random(0)
203 | val indexes = {
204 | val M = N / 10
205 | val a = new Array[Int](M)
206 | var i = 0
207 | while (i < M) {
208 | a(i) = r.nextInt(N)
209 | i += 1
210 | }
211 | a
212 | }
213 | val R = 5
214 | time("random access performance", repeat = 2, blockRepeat = R) {
215 | block("scala array") {
216 | for (i <- indexes)
217 | arr1(i) = 1
218 | }
219 |
220 | block("LIntArray") {
221 | for (i <- indexes)
222 | arr2(i) = 1
223 |
224 | }
225 |
226 | block("LIntArraySimple") {
227 | for (i <- indexes)
228 | arr3(i) = 1
229 | }
230 |
231 | block("MatrixBasedLIntArray") {
232 | for (i <- indexes)
233 | arr4(i) = 1
234 | }
235 | }
236 | }
237 | finally {
238 | arr2.free
239 | arr3.free
240 | }
241 | }
242 |
243 | "compare sequential access performance" in {
244 |
245 | //val N = 1 * 1024 * 1024 * 1024
246 | val N = 64 * 1024 * 1024
247 | info("benchmark has started..")
248 | val arr1 = new Array[Int](N)
249 | val arr2 = new LIntArray(N)
250 | val arr3 = new LIntArraySimple(N)
251 | val arr4 = new MatrixBasedLIntArray(N)
252 |
253 | try {
254 | val range = (0 until (N / 10)).map(_.toLong).toSeq
255 | time("sequential read performance", repeat = 5) {
256 | block("scala array") {
257 | for (i <- range)
258 | arr1(i.toInt)
259 | }
260 |
261 | block("LIntArray") {
262 | for (i <- range)
263 | arr2(i)
264 |
265 | }
266 |
267 | block("LIntArraySimple") {
268 | for (i <- range)
269 | arr3(i)
270 | }
271 |
272 | block("MatrixBasedLIntArray") {
273 | for (i <- range)
274 | arr4(i)
275 | }
276 |
277 | }
278 | }
279 | finally {
280 | arr2.free
281 | arr3.free
282 | }
283 | }
284 |
285 |
286 | "create large array" taggedAs ("la") in {
287 | info("large memory allocation test")
288 | for (i <- 0 until 10) {
289 | val arr = new LByteArray(2L * G)
290 | try {
291 | arr(arr.size - 1) = 134.toByte
292 | }
293 | finally {
294 | arr.free
295 | }
296 | }
297 | }
298 |
299 | "create view" taggedAs("view") in {
300 | val l = LArray(1, 13, 4, 5)
301 | val v = l.view(1, 3)
302 | v.mkString(", ") shouldBe "13, 4"
303 | val b = v.toDirectByteBuffer(0)
304 | b.getInt() shouldBe 13
305 | b.getInt() shouldBe 4
306 | }
307 |
308 | }
309 |
310 |
311 |
312 | "LByteArray" should {
313 |
314 | "have constructor" in {
315 | val a = Array[Byte](1, 2, 3)
316 |
317 | val b = LArray[Byte](1, 5, 34)
318 |
319 | b(0) should be(1.toByte)
320 | b(1) should be(5.toByte)
321 | b(2) should be(34.toByte)
322 | }
323 |
324 | "compare performance" taggedAs ("bp") in {
325 |
326 | val N = (0.01 * G).toLong
327 | val a = new Array[Byte](N.toInt)
328 | val b = new LByteArray(N)
329 | info("LByteArray performance test has started")
330 | time("LByteArray random access & sort", repeat = 5) {
331 | block("native array") {
332 | val r = new Random(0)
333 | for (i <- 0L until N) {
334 | val index = (i / 4) * 4
335 | a((index + (i % 4L)).toInt) = r.nextInt.toByte
336 | }
337 | java.util.Arrays.sort(a)
338 | }
339 |
340 | block("LByteArray") {
341 | val r = new Random(0)
342 | for (i <- 0L until N) {
343 | val index = (i / 4) * 4
344 | b((index + (i % 4L))) = r.nextInt.toByte
345 | }
346 | b.sort
347 | }
348 |
349 | }
350 |
351 | info("sequential access test")
352 | time("LByteArray sequential write", repeat = 5) {
353 | block("native array") {
354 | val r = new Random(0)
355 | for (i <- 0L until N) {
356 | a(i.toInt) = r.nextInt.toByte
357 | }
358 | }
359 |
360 | block("LByteArray") {
361 | val r = new Random(0)
362 | for (i <- 0L until N) {
363 | b(i) = r.nextInt.toByte
364 | }
365 | }
366 | }
367 |
368 | b.free
369 | }
370 |
371 |
372 | }
373 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LArrayTestWithPBT.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | package xerial.larray
17 |
18 | import org.scalatest.prop.PropertyChecks
19 | import org.scalatest.prop.Checkers
20 |
21 | /**
22 | * Created with IntelliJ IDEA.
23 | * User: hayato
24 | * Date: 13/03/27
25 | * Time: 15:06
26 | */
27 | class LArrayTestWithPBT
28 | extends LArraySpec with PropertyChecks with LArrayBehaviour {
29 |
30 | implicit val config = PropertyCheckConfig(minSuccessful = 3, minSize = 1, maxSize = 10000)
31 |
32 | forAll {
33 | (input: Array[Int]) =>
34 | s"int array [${input.take(10).mkString(", ")}, ...]" should {
35 | behave like validArray(input)
36 | behave like validIntArray(input)
37 | }
38 | }
39 |
40 | forAll {
41 | (input: Array[Long]) =>
42 | s"long array [${input.take(10).mkString(", ")}, ...]" should {
43 | behave like validArray(input)
44 | behave like validLongArray(input)
45 | }
46 | }
47 |
48 | forAll {
49 | (input: Array[Float]) =>
50 | s"float array [${input.take(10).mkString(", ")}, ...]" should {
51 | behave like validArray(input)
52 | behave like validFloatArray(input)
53 | }
54 | }
55 |
56 | forAll {
57 | (input: Array[Double]) =>
58 | s"double array [${input.take(10).mkString(", ")}, ...]" should {
59 | behave like validArray(input)
60 | behave like validDoubleArray(input)
61 | }
62 | }
63 |
64 | "empty test" should {
65 | val input = Seq.empty[Int]
66 | behave like validArray(input)
67 | behave like validIntArray(input)
68 | }
69 |
70 | }
71 |
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/LBitArrayTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LBitArrayTest.scala
19 | // Since: 2013/03/25 12:33
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import scala.util.Random
26 |
27 | /**
28 | * @author Taro L. Saito
29 | */
30 | class LBitArrayTest extends LArraySpec with LArrayBehaviour {
31 | "LBitArray" should {
32 |
33 | "have constructor" in {
34 | val b = new LBitArray(6)
35 | b.size should be (6)
36 |
37 | b.clear
38 | b.on(1)
39 | b.toString should be ("010000")
40 |
41 | b.on(5)
42 | b.toString should be ("010001")
43 | }
44 |
45 | "set bits" in {
46 | val N = 100
47 | val b = new LBitArray(N)
48 | b.size should be (N)
49 | debug(b)
50 | b.clear
51 | debug(b)
52 | b.forall(_ == false) should be (true)
53 | for(i <- 0L until b.length) {
54 | b.on(i)
55 | }
56 | b.forall(_ == true) should be (true)
57 | }
58 |
59 | "on and off specific bits" in {
60 | val b = new LBitArray(10000)
61 | b.fill
62 | b.forall(_ == true) should be (true)
63 |
64 | for(pos <- Seq(91, 34, 5093, 443, 4)) {
65 | b.off(pos)
66 | b(pos) should be (false)
67 | }
68 | }
69 |
70 | "have builder" in {
71 |
72 | val b = LArray.newBuilder[Boolean]
73 | val in = Seq(true, false, false, true, true, false, false, true, true)
74 | in.foreach( b += _ )
75 | val l = b.result()
76 |
77 | debug(l)
78 |
79 | l.toString should be (in.map(v => if(v) "1" else "0").mkString)
80 |
81 | }
82 |
83 | "behave like valid LArray" should {
84 | val input = Seq(true, false, true, false, false, false, true, true)
85 | behave like validArray(input)
86 | }
87 |
88 | "behave like valid LArray for large input" should {
89 | val input2 = (for(i <- 0 until 150) yield { Random.nextBoolean }).toArray.toSeq
90 | behave like validArray(input2)
91 | }
92 |
93 | }
94 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/MappedLByteArrayTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // MappedLByteArrayTest.scala
19 | // Since: 2013/04/04 1:58 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.File
26 |
27 | /**
28 | * @author Taro L. Saito
29 | */
30 | class MappedLByteArrayTest extends LArraySpec {
31 |
32 | "MappedLByteArray" should {
33 |
34 | "create memory mapped file" taggedAs ("simple") in {
35 |
36 | import xerial.larray._
37 |
38 | val f = File.createTempFile("mmap", ".larray", new File("target"))
39 | f.deleteOnExit()
40 |
41 | val L = 100L
42 | val m = new MappedLByteArray(f, 0, L)
43 | m.size shouldBe L
44 | for (i <- 0 Until m.size) {
45 | m(i) = i.toByte
46 | }
47 |
48 | trace(m.mkString(", "))
49 | val mc = m.slice(0)
50 | m.close()
51 |
52 | val m2 = LArray.loadFrom[Byte](f)
53 | mc.sameElements(m2) should be(true)
54 |
55 | val mOffset = new MappedLByteArray(f, 3, L - 3)
56 | trace(mOffset.mkString(", "))
57 |
58 | mc.slice(3).sameElements(mOffset) should be(true)
59 |
60 | //mOffset.flush
61 | //m.free
62 | //m2.free
63 | }
64 |
65 | "create large memory mapped file more than 2GB" taggedAs ("large") in {
66 |
67 | val f = File.createTempFile("mmap", ".larray", new File("target"))
68 | f.deleteOnExit()
69 |
70 | val G = 1024L * 1024 * 1024
71 | val m = new MappedLByteArray(f, 0, 2L * G + 1024)
72 | val offset = 100
73 |
74 | val v = 34.toByte
75 | m(2L * G + offset) = v
76 | m.close()
77 |
78 | f.length() shouldBe m.size
79 |
80 | val view = new MappedLByteArray(f, 2L * G, 1024)
81 | view(offset) shouldBe v
82 | view.close()
83 |
84 | f.delete()
85 | }
86 |
87 | }
88 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/SnappyCompressTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // JSnappyCompressTest.scala
19 | // Since: 2013/03/27 16:17
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | import java.io.File
26 | import xerial.larray.mmap.MMapMode
27 | import org.xerial.snappy.Snappy
28 | import xerial.larray.buffer.LBuffer
29 |
30 | /**
31 | * @author Taro L. Saito
32 | */
33 | class SnappyCompressTest extends LArraySpec {
34 |
35 | implicit class AsRaw[A](l:LArray[A]) {
36 | def address = l.asInstanceOf[RawByteArray[Int]].address
37 | }
38 |
39 | "Snappy" should {
40 |
41 | import xerial.larray._
42 |
43 | "support zero-copy compress with LBuffer" taggedAs("zerocopy") in {
44 | val N = 10000
45 | val buf = new LBuffer(4 * N)
46 | for(i <- 0 Until N) {
47 | buf.putFloat(i * 4, math.sin(i * 0.01).toFloat)
48 | }
49 |
50 | val arr = buf.toArray
51 | val bufSize = buf.size.toInt
52 |
53 |
54 | val maxCompressedLength = Snappy.maxCompressedLength(bufSize)
55 |
56 | val GN = 10
57 | val R = 5
58 |
59 | info("Start compression benchmark")
60 | time("compress", repeat = GN, blockRepeat = R) {
61 | block("LBuffer -> LBuffer (raw)") {
62 | val out = new LBuffer(maxCompressedLength)
63 | Snappy.rawCompress(buf.address, bufSize, out.address)
64 | }
65 |
66 | block("Array -> Array (raw) ") {
67 | val out = new Array[Byte](maxCompressedLength)
68 | Snappy.rawCompress(arr, 0, bufSize, out, 0)
69 | }
70 |
71 | block("Array -> Array (dain)") {
72 | val out = new Array[Byte](maxCompressedLength)
73 | org.iq80.snappy.Snappy.compress(arr, 0, bufSize, out, 0)
74 | }
75 | }
76 |
77 | val compressedLBuffer = new LBuffer(maxCompressedLength)
78 | val compressedArray = new Array[Byte](maxCompressedLength)
79 | val compressedArrayDain = new Array[Byte](maxCompressedLength)
80 |
81 | Snappy.rawCompress(buf.address, bufSize, compressedLBuffer.address)
82 | Snappy.rawCompress(arr, 0, bufSize, compressedArray, 0)
83 | org.iq80.snappy.Snappy.compress(arr, 0, bufSize, compressedArrayDain, 0)
84 |
85 | val compressedSize = Snappy.compress(buf.toArray).length
86 |
87 | time("decompress", repeat = GN, blockRepeat = R) {
88 | block("LBuffer -> LBuffer (raw)") {
89 | val out = new LBuffer(bufSize)
90 | Snappy.rawUncompress(compressedLBuffer.address, compressedSize, out.address)
91 | }
92 |
93 | block("Array -> Array (raw) ") {
94 | val out = new Array[Byte](bufSize)
95 | Snappy.rawUncompress(compressedArray, 0, compressedSize, out, 0)
96 | }
97 |
98 | block("Array -> Array (dain)") {
99 | val out = new Array[Byte](bufSize)
100 | org.iq80.snappy.Snappy.uncompress(compressedArrayDain, 0, compressedSize, out, 0)
101 | }
102 |
103 | }
104 |
105 | }
106 |
107 |
108 | "compress LArray" in {
109 | val l = (for (i <- 0 until 3000) yield math.toDegrees(math.sin(i/360)).toInt).toLArray
110 | val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt)
111 | val compressedBuf = LArray.of[Byte](maxLen)
112 | val compressedLen = Snappy.rawCompress(l.address, l.byteLength, compressedBuf.address)
113 |
114 | val compressed = compressedBuf.slice(0, compressedLen)
115 | val uncompressedLength = Snappy.uncompressedLength(compressed.address, compressed.byteLength)
116 | val uncompressed = LArray.of[Int](uncompressedLength / 4)
117 | Snappy.rawUncompress(compressed.address, compressed.byteLength, uncompressed.address)
118 |
119 | debug(s"byteLength:${l.byteLength}, max compressed length:$maxLen ,compressed length:$compressedLen")
120 | l.sameElements(uncompressed) should be (true)
121 | }
122 |
123 | "compress LIntArray" taggedAs("it") in {
124 | val N = 100000000L
125 | val l = new LIntArray(N)
126 | info(f"preparing data set. N=$N%,d")
127 | for(i <- 0 Until N) l(i) = math.toDegrees(math.sin(i/360)).toInt
128 |
129 | debug("compressing the data")
130 | val maxLen = Snappy.maxCompressedLength(l.byteLength.toInt)
131 | val compressedBuf = LArray.of[Byte](maxLen)
132 | val compressedLen = Snappy.rawCompress(l.address, l.byteLength, compressedBuf.address)
133 | val compressed = compressedBuf.slice(0, compressedLen)
134 | val f = File.createTempFile("snappy", ".dat", new File("target"))
135 | f.deleteOnExit()
136 | compressed.saveTo(f)
137 |
138 | debug("decompressing the data")
139 | val b = LArray.mmap(f, MMapMode.READ_ONLY)
140 | val len = Snappy.uncompressedLength(b.address, b.length)
141 | val decompressed = new LIntArray(len / 4)
142 | Snappy.rawUncompress(b.address, b.length, decompressed.address)
143 | b.close
144 |
145 | debug(f"l.length:${l.length}%,d, decompressed.length:${decompressed.length}%,d")
146 |
147 | l.sameElements(decompressed) should be (true)
148 | info("start bench")
149 | time("iterate", repeat=10) {
150 | block("new array") {
151 | var sum = 0L
152 | var i = 0
153 | while(i < l.length) {
154 | sum += l(i)
155 | i += 1
156 | }
157 | }
158 |
159 | block("decompressed") {
160 | var sum = 0L
161 | var i = 0
162 | while(i < decompressed.length) {
163 | sum += decompressed(i)
164 | i += 1
165 | }
166 | }
167 | }
168 | }
169 | }
170 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/UInt32ArrayTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // UInt32ArrayTest.scala
19 | // Since: 2013/03/18 4:08 PM
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray
24 |
25 | /**
26 | * @author Taro L. Saito
27 | */
28 | class UInt32ArrayTest extends LArraySpec {
29 |
30 | "UInt32Array" should {
31 |
32 | "record values larger than 2G" in {
33 |
34 | val u = new UInt32Array(10)
35 | val v : Long = Int.MaxValue.toLong + 10L
36 | u(0) = v
37 | u(0) should be (v)
38 |
39 | u(1) = 3141341
40 | u(1) should be (3141341L)
41 |
42 | val v2 = Integer.MAX_VALUE.toLong * 2L
43 | u(2) = v2
44 | u(2) should be (v2)
45 | }
46 |
47 | "have builder" in {
48 | val b = UInt32Array.newBuilder
49 | b += 1
50 | b += Int.MaxValue.toLong + 3L
51 | b += 4
52 | val u = b.result
53 |
54 | u.size should be(3)
55 | u(0) should be (1L)
56 | u(1) should be (Int.MaxValue.toLong + 3L)
57 | u(2) should be (4L)
58 |
59 | }
60 |
61 | }
62 |
63 | }
--------------------------------------------------------------------------------
/larray/src/test/scala/xerial/larray/example/LArrayExampleTest.scala:
--------------------------------------------------------------------------------
1 | /*--------------------------------------------------------------------------
2 | * Copyright 2013 Taro L. Saito
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *--------------------------------------------------------------------------*/
16 | //--------------------------------------
17 | //
18 | // LArrayExampleTest.scala
19 | // Since: 2013/03/26 15:04
20 | //
21 | //--------------------------------------
22 |
23 | package xerial.larray.example
24 |
25 | import xerial.larray.LArraySpec
26 |
27 |
28 | /**
29 | * @author Taro L. Saito
30 | */
31 | class LArrayExampleTest extends LArraySpec {
32 |
33 | "LArrayExample" should {
34 | "run Scala API" in {
35 | val out = captureOut(new LArrayExample)
36 | out should (include ("done."))
37 | }
38 |
39 | "run Java API" in {
40 | val out = captureSystemOut(LArrayJavaExample.main(Array.empty[String]))
41 | out should (include ("done."))
42 | }
43 |
44 | }
45 | }
--------------------------------------------------------------------------------
/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version=0.13.13
2 |
--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.7.9")
2 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
3 | addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M15")
4 | addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3")
5 | addSbtPlugin("com.typesafe.sbt" % "sbt-multi-jvm" % "0.3.11")
6 | addSbtPlugin("com.eed3si9n" % "sbt-doge" % "0.1.5")
7 |
--------------------------------------------------------------------------------
/version.sbt:
--------------------------------------------------------------------------------
1 | version in ThisBuild := "0.4.2-SNAPSHOT"
2 |
--------------------------------------------------------------------------------